1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication.regionserver;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HBaseConfiguration;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.KeyValue;
29  import org.apache.hadoop.hbase.client.Get;
30  import org.apache.hadoop.hbase.client.HTable;
31  import org.apache.hadoop.hbase.client.Result;
32  import org.apache.hadoop.hbase.client.ResultScanner;
33  import org.apache.hadoop.hbase.client.Scan;
34  import org.apache.hadoop.hbase.regionserver.wal.HLog;
35  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
36  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.junit.AfterClass;
39  import org.junit.Before;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  
43  import java.util.concurrent.atomic.AtomicBoolean;
44  
45  import static org.junit.Assert.assertEquals;
46  import static org.junit.Assert.assertTrue;
47  
48  public class TestReplicationSink {
49  
50    private static final Log LOG =
51        LogFactory.getLog(TestReplicationSink.class);
52  
53    private static final int BATCH_SIZE = 10;
54  
55    private static final long SLEEP_TIME = 500;
56  
57    private final static Configuration conf = HBaseConfiguration.create();
58  
59    private final static HBaseTestingUtility TEST_UTIL =
60        new HBaseTestingUtility();
61  
62    private static ReplicationSink SINK;
63  
64    private static final byte[] TABLE_NAME1 =
65        Bytes.toBytes("table1");
66    private static final byte[] TABLE_NAME2 =
67        Bytes.toBytes("table2");
68  
69    private static final byte[] FAM_NAME1 = Bytes.toBytes("info1");
70    private static final byte[] FAM_NAME2 = Bytes.toBytes("info2");
71  
72    private static final AtomicBoolean STOPPER = new AtomicBoolean(false);
73  
74    private static HTable table1;
75  
76    private static HTable table2;
77  
78     /**
79     * @throws java.lang.Exception
80     */
81    @BeforeClass
82    public static void setUpBeforeClass() throws Exception {
83      TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
84      TEST_UTIL.getConfiguration().setBoolean(
85          HConstants.REPLICATION_ENABLE_KEY, true);
86      TEST_UTIL.startMiniCluster(3);
87      conf.setBoolean("dfs.support.append", true);
88      SINK = new ReplicationSink(conf,STOPPER);
89      table1 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAME1);
90      table2 = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAME2);
91    }
92  
93    /**
94     * @throws java.lang.Exception
95     */
96    @AfterClass
97    public static void tearDownAfterClass() throws Exception {
98      STOPPER.set(true);
99      TEST_UTIL.shutdownMiniCluster();
100   }
101 
102   /**
103    * @throws java.lang.Exception
104    */
105   @Before
106   public void setUp() throws Exception {
107     table1 = TEST_UTIL.truncateTable(TABLE_NAME1);
108     table2 = TEST_UTIL.truncateTable(TABLE_NAME2);
109     Thread.sleep(SLEEP_TIME);
110   }
111 
112   /**
113    * Insert a whole batch of entries
114    * @throws Exception
115    */
116   @Test
117   public void testBatchSink() throws Exception {
118     HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE];
119     for(int i = 0; i < BATCH_SIZE; i++) {
120       entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
121     }
122     SINK.replicateEntries(entries);
123     Scan scan = new Scan();
124     ResultScanner scanRes = table1.getScanner(scan);
125     assertEquals(BATCH_SIZE, scanRes.next(BATCH_SIZE).length);
126   }
127 
128   /**
129    * Insert a mix of puts and deletes
130    * @throws Exception
131    */
132   @Test
133   public void testMixedPutDelete() throws Exception {
134     HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE/2];
135     for(int i = 0; i < BATCH_SIZE/2; i++) {
136       entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
137     }
138     SINK.replicateEntries(entries);
139 
140     entries = new HLog.Entry[BATCH_SIZE];
141     for(int i = 0; i < BATCH_SIZE; i++) {
142       entries[i] = createEntry(TABLE_NAME1, i,
143           i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn);
144     }
145 
146     SINK.replicateEntries(entries);
147     Scan scan = new Scan();
148     ResultScanner scanRes = table1.getScanner(scan);
149     assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length);
150   }
151 
152   /**
153    * Insert to 2 different tables
154    * @throws Exception
155    */
156   @Test
157   public void testMixedPutTables() throws Exception {
158     HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE];
159     for(int i = 0; i < BATCH_SIZE; i++) {
160       entries[i] =
161           createEntry( i % 2 == 0 ? TABLE_NAME2 : TABLE_NAME1,
162               i, KeyValue.Type.Put);
163     }
164 
165     SINK.replicateEntries(entries);
166     Scan scan = new Scan();
167     ResultScanner scanRes = table2.getScanner(scan);
168     for(Result res : scanRes) {
169       assertTrue(Bytes.toInt(res.getRow()) % 2 == 0);
170     }
171   }
172 
173   /**
174    * Insert then do different types of deletes
175    * @throws Exception
176    */
177   @Test
178   public void testMixedDeletes() throws Exception {
179     HLog.Entry[] entries = new HLog.Entry[3];
180     for(int i = 0; i < 3; i++) {
181       entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
182     }
183     SINK.replicateEntries(entries);
184     entries = new HLog.Entry[3];
185 
186     entries[0] = createEntry(TABLE_NAME1, 0, KeyValue.Type.DeleteColumn);
187     entries[1] = createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily);
188     entries[2] = createEntry(TABLE_NAME1, 2, KeyValue.Type.DeleteColumn);
189 
190     SINK.replicateEntries(entries);
191 
192     Scan scan = new Scan();
193     ResultScanner scanRes = table1.getScanner(scan);
194     assertEquals(0, scanRes.next(3).length);
195   }
196 
197   /**
198    * Puts are buffered, but this tests when a delete (not-buffered) is applied
199    * before the actual Put that creates it.
200    * @throws Exception
201    */
202   @Test
203   public void testApplyDeleteBeforePut() throws Exception {
204     HLog.Entry[] entries = new HLog.Entry[5];
205     for(int i = 0; i < 2; i++) {
206       entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
207     }
208     entries[2] = createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily);
209     for(int i = 3; i < 5; i++) {
210       entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
211     }
212     SINK.replicateEntries(entries);
213     Get get = new Get(Bytes.toBytes(1));
214     Result res = table1.get(get);
215     assertEquals(0, res.size());
216   }
217 
218   private HLog.Entry createEntry(byte [] table, int row,  KeyValue.Type type) {
219     byte[] fam = Bytes.equals(table, TABLE_NAME1) ? FAM_NAME1 : FAM_NAME2;
220     byte[] rowBytes = Bytes.toBytes(row);
221     // Just make sure we don't get the same ts for two consecutive rows with
222     // same key
223     try {
224       Thread.sleep(1);
225     } catch (InterruptedException e) {
226       LOG.info("Was interrupted while sleep, meh", e);
227     }
228     final long now = System.currentTimeMillis();
229     KeyValue kv = null;
230     if(type.getCode() == KeyValue.Type.Put.getCode()) {
231       kv = new KeyValue(rowBytes, fam, fam, now,
232           KeyValue.Type.Put, Bytes.toBytes(row));
233     } else if (type.getCode() == KeyValue.Type.DeleteColumn.getCode()) {
234         kv = new KeyValue(rowBytes, fam, fam,
235             now, KeyValue.Type.DeleteColumn);
236     } else if (type.getCode() == KeyValue.Type.DeleteFamily.getCode()) {
237         kv = new KeyValue(rowBytes, fam, null,
238             now, KeyValue.Type.DeleteFamily);
239     }
240 
241     HLogKey key = new HLogKey(table, table, now, now);
242 
243     WALEdit edit = new WALEdit();
244     edit.add(kv);
245 
246     return new HLog.Entry(key, edit);
247   }
248 }