1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.replication.regionserver;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.HBaseConfiguration;
26 import org.apache.hadoop.hbase.HBaseTestingUtility;
27 import org.apache.hadoop.hbase.HConstants;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.client.Get;
30 import org.apache.hadoop.hbase.client.HTable;
31 import org.apache.hadoop.hbase.client.Result;
32 import org.apache.hadoop.hbase.client.ResultScanner;
33 import org.apache.hadoop.hbase.client.Scan;
34 import org.apache.hadoop.hbase.regionserver.wal.HLog;
35 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
36 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
37 import org.apache.hadoop.hbase.util.Bytes;
38 import org.junit.AfterClass;
39 import org.junit.Before;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42
43 import java.util.concurrent.atomic.AtomicBoolean;
44
45 import static org.junit.Assert.assertEquals;
46 import static org.junit.Assert.assertTrue;
47
48 public class TestReplicationSink {
49
50 private static final Log LOG =
51 LogFactory.getLog(TestReplicationSink.class);
52
53 private static final int BATCH_SIZE = 10;
54
55 private static final long SLEEP_TIME = 500;
56
57 private final static Configuration conf = HBaseConfiguration.create();
58
59 private final static HBaseTestingUtility TEST_UTIL =
60 new HBaseTestingUtility();
61
62 private static ReplicationSink SINK;
63
64 private static final byte[] TABLE_NAME1 =
65 Bytes.toBytes("table1");
66 private static final byte[] TABLE_NAME2 =
67 Bytes.toBytes("table2");
68
69 private static final byte[] FAM_NAME1 = Bytes.toBytes("info1");
70 private static final byte[] FAM_NAME2 = Bytes.toBytes("info2");
71
72 private static final AtomicBoolean STOPPER = new AtomicBoolean(false);
73
74 private static HTable table1;
75
76 private static HTable table2;
77
78
79
80
81 @BeforeClass
82 public static void setUpBeforeClass() throws Exception {
83 TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
84 TEST_UTIL.getConfiguration().setBoolean(
85 HConstants.REPLICATION_ENABLE_KEY, true);
86 TEST_UTIL.startMiniCluster(3);
87 conf.setBoolean("dfs.support.append", true);
88 SINK = new ReplicationSink(conf,STOPPER);
89 table1 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAME1);
90 table2 = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAME2);
91 }
92
93
94
95
96 @AfterClass
97 public static void tearDownAfterClass() throws Exception {
98 STOPPER.set(true);
99 TEST_UTIL.shutdownMiniCluster();
100 }
101
102
103
104
105 @Before
106 public void setUp() throws Exception {
107 table1 = TEST_UTIL.truncateTable(TABLE_NAME1);
108 table2 = TEST_UTIL.truncateTable(TABLE_NAME2);
109 Thread.sleep(SLEEP_TIME);
110 }
111
112
113
114
115
116 @Test
117 public void testBatchSink() throws Exception {
118 HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE];
119 for(int i = 0; i < BATCH_SIZE; i++) {
120 entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
121 }
122 SINK.replicateEntries(entries);
123 Scan scan = new Scan();
124 ResultScanner scanRes = table1.getScanner(scan);
125 assertEquals(BATCH_SIZE, scanRes.next(BATCH_SIZE).length);
126 }
127
128
129
130
131
132 @Test
133 public void testMixedPutDelete() throws Exception {
134 HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE/2];
135 for(int i = 0; i < BATCH_SIZE/2; i++) {
136 entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
137 }
138 SINK.replicateEntries(entries);
139
140 entries = new HLog.Entry[BATCH_SIZE];
141 for(int i = 0; i < BATCH_SIZE; i++) {
142 entries[i] = createEntry(TABLE_NAME1, i,
143 i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn);
144 }
145
146 SINK.replicateEntries(entries);
147 Scan scan = new Scan();
148 ResultScanner scanRes = table1.getScanner(scan);
149 assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length);
150 }
151
152
153
154
155
156 @Test
157 public void testMixedPutTables() throws Exception {
158 HLog.Entry[] entries = new HLog.Entry[BATCH_SIZE];
159 for(int i = 0; i < BATCH_SIZE; i++) {
160 entries[i] =
161 createEntry( i % 2 == 0 ? TABLE_NAME2 : TABLE_NAME1,
162 i, KeyValue.Type.Put);
163 }
164
165 SINK.replicateEntries(entries);
166 Scan scan = new Scan();
167 ResultScanner scanRes = table2.getScanner(scan);
168 for(Result res : scanRes) {
169 assertTrue(Bytes.toInt(res.getRow()) % 2 == 0);
170 }
171 }
172
173
174
175
176
177 @Test
178 public void testMixedDeletes() throws Exception {
179 HLog.Entry[] entries = new HLog.Entry[3];
180 for(int i = 0; i < 3; i++) {
181 entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
182 }
183 SINK.replicateEntries(entries);
184 entries = new HLog.Entry[3];
185
186 entries[0] = createEntry(TABLE_NAME1, 0, KeyValue.Type.DeleteColumn);
187 entries[1] = createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily);
188 entries[2] = createEntry(TABLE_NAME1, 2, KeyValue.Type.DeleteColumn);
189
190 SINK.replicateEntries(entries);
191
192 Scan scan = new Scan();
193 ResultScanner scanRes = table1.getScanner(scan);
194 assertEquals(0, scanRes.next(3).length);
195 }
196
197
198
199
200
201
202 @Test
203 public void testApplyDeleteBeforePut() throws Exception {
204 HLog.Entry[] entries = new HLog.Entry[5];
205 for(int i = 0; i < 2; i++) {
206 entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
207 }
208 entries[2] = createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily);
209 for(int i = 3; i < 5; i++) {
210 entries[i] = createEntry(TABLE_NAME1, i, KeyValue.Type.Put);
211 }
212 SINK.replicateEntries(entries);
213 Get get = new Get(Bytes.toBytes(1));
214 Result res = table1.get(get);
215 assertEquals(0, res.size());
216 }
217
218 private HLog.Entry createEntry(byte [] table, int row, KeyValue.Type type) {
219 byte[] fam = Bytes.equals(table, TABLE_NAME1) ? FAM_NAME1 : FAM_NAME2;
220 byte[] rowBytes = Bytes.toBytes(row);
221
222
223 try {
224 Thread.sleep(1);
225 } catch (InterruptedException e) {
226 LOG.info("Was interrupted while sleep, meh", e);
227 }
228 final long now = System.currentTimeMillis();
229 KeyValue kv = null;
230 if(type.getCode() == KeyValue.Type.Put.getCode()) {
231 kv = new KeyValue(rowBytes, fam, fam, now,
232 KeyValue.Type.Put, Bytes.toBytes(row));
233 } else if (type.getCode() == KeyValue.Type.DeleteColumn.getCode()) {
234 kv = new KeyValue(rowBytes, fam, fam,
235 now, KeyValue.Type.DeleteColumn);
236 } else if (type.getCode() == KeyValue.Type.DeleteFamily.getCode()) {
237 kv = new KeyValue(rowBytes, fam, null,
238 now, KeyValue.Type.DeleteFamily);
239 }
240
241 HLogKey key = new HLogKey(table, table, now, now);
242
243 WALEdit edit = new WALEdit();
244 edit.add(kv);
245
246 return new HLog.Entry(key, edit);
247 }
248 }