1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.replication;
21
22 import static org.junit.Assert.assertArrayEquals;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.List;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.hbase.*;
33 import org.apache.hadoop.hbase.client.Delete;
34 import org.apache.hadoop.hbase.client.Get;
35 import org.apache.hadoop.hbase.client.HBaseAdmin;
36 import org.apache.hadoop.hbase.client.HTable;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
40 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
41 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
42 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
43 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
46 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
47 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
48 import org.junit.After;
49 import org.junit.Before;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53 @Category(LargeTests.class)
54 public class TestMasterReplication {
55
56 private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
57
58 private Configuration conf1;
59 private Configuration conf2;
60 private Configuration conf3;
61
62 private HBaseTestingUtility utility1;
63 private HBaseTestingUtility utility2;
64 private HBaseTestingUtility utility3;
65
66 private MiniZooKeeperCluster miniZK;
67
68 private static final long SLEEP_TIME = 500;
69 private static final int NB_RETRIES = 10;
70
71 private static final byte[] tableName = Bytes.toBytes("test");
72 private static final byte[] famName = Bytes.toBytes("f");
73 private static final byte[] row = Bytes.toBytes("row");
74 private static final byte[] row1 = Bytes.toBytes("row1");
75 private static final byte[] row2 = Bytes.toBytes("row2");
76 private static final byte[] noRepfamName = Bytes.toBytes("norep");
77
78 private static final byte[] count = Bytes.toBytes("count");
79 private static final byte[] put = Bytes.toBytes("put");
80 private static final byte[] delete = Bytes.toBytes("delete");
81
82 private HTableDescriptor table;
83
84 @Before
85 public void setUp() throws Exception {
86 conf1 = HBaseConfiguration.create();
87 conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
88
89
90 conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20);
91 conf1.setInt("replication.source.size.capacity", 1024);
92 conf1.setLong("replication.source.sleepforretries", 100);
93 conf1.setInt("hbase.regionserver.maxlogs", 10);
94 conf1.setLong("hbase.master.logcleaner.ttl", 10);
95 conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
96 conf1.setBoolean("dfs.support.append", true);
97 conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
98 conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
99 CoprocessorCounter.class.getName());
100
101 utility1 = new HBaseTestingUtility(conf1);
102 utility1.startMiniZKCluster();
103 miniZK = utility1.getZkCluster();
104
105
106
107 utility1.setZkCluster(miniZK);
108 new ZooKeeperWatcher(conf1, "cluster1", null, true);
109
110 conf2 = new Configuration(conf1);
111 conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
112
113 utility2 = new HBaseTestingUtility(conf2);
114 utility2.setZkCluster(miniZK);
115 new ZooKeeperWatcher(conf2, "cluster2", null, true);
116
117 conf3 = new Configuration(conf1);
118 conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");
119
120 utility3 = new HBaseTestingUtility(conf3);
121 utility3.setZkCluster(miniZK);
122 new ZooKeeperWatcher(conf3, "cluster3", null, true);
123
124 table = new HTableDescriptor(tableName);
125 HColumnDescriptor fam = new HColumnDescriptor(famName);
126 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
127 table.addFamily(fam);
128 fam = new HColumnDescriptor(noRepfamName);
129 table.addFamily(fam);
130 }
131
132 @After
133 public void tearDown() throws IOException {
134 miniZK.shutdown();
135 }
136
137 @Test(timeout=300000)
138 public void testCyclicReplication() throws Exception {
139 LOG.info("testCyclicReplication");
140 utility1.startMiniCluster();
141 utility2.startMiniCluster();
142 utility3.startMiniCluster();
143 ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
144 ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
145 ReplicationAdmin admin3 = new ReplicationAdmin(conf3);
146
147 new HBaseAdmin(conf1).createTable(table);
148 new HBaseAdmin(conf2).createTable(table);
149 new HBaseAdmin(conf3).createTable(table);
150 HTable htable1 = new HTable(conf1, tableName);
151 htable1.setWriteBufferSize(1024);
152 HTable htable2 = new HTable(conf2, tableName);
153 htable2.setWriteBufferSize(1024);
154 HTable htable3 = new HTable(conf3, tableName);
155 htable3.setWriteBufferSize(1024);
156
157 admin1.addPeer("1", utility2.getClusterKey());
158 admin2.addPeer("1", utility3.getClusterKey());
159 admin3.addPeer("1", utility1.getClusterKey());
160
161
162 putAndWait(row, famName, htable1, htable3);
163
164 check(row,famName,htable2);
165
166 putAndWait(row1, famName, htable2, htable1);
167 check(row,famName,htable3);
168 putAndWait(row2, famName, htable3, htable2);
169 check(row,famName,htable1);
170
171 deleteAndWait(row,htable1,htable3);
172 deleteAndWait(row1,htable2,htable1);
173 deleteAndWait(row2,htable3,htable2);
174
175 assertEquals("Puts were replicated back ", 3, getCount(htable1, put));
176 assertEquals("Puts were replicated back ", 3, getCount(htable2, put));
177 assertEquals("Puts were replicated back ", 3, getCount(htable3, put));
178 assertEquals("Deletes were replicated back ", 3, getCount(htable1, delete));
179 assertEquals("Deletes were replicated back ", 3, getCount(htable2, delete));
180 assertEquals("Deletes were replicated back ", 3, getCount(htable3, delete));
181 utility3.shutdownMiniCluster();
182 utility2.shutdownMiniCluster();
183 utility1.shutdownMiniCluster();
184 }
185
186
187
188
189
190
191
192 @Test(timeout=300000)
193 public void testSimplePutDelete() throws Exception {
194 LOG.info("testSimplePutDelete");
195 utility1.startMiniCluster();
196 utility2.startMiniCluster();
197
198 ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
199 ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
200
201 new HBaseAdmin(conf1).createTable(table);
202 new HBaseAdmin(conf2).createTable(table);
203 HTable htable1 = new HTable(conf1, tableName);
204 htable1.setWriteBufferSize(1024);
205 HTable htable2 = new HTable(conf2, tableName);
206 htable2.setWriteBufferSize(1024);
207
208
209 admin1.addPeer("1", utility2.getClusterKey());
210 admin2.addPeer("1", utility1.getClusterKey());
211
212
213
214 putAndWait(row, famName, htable1, htable2);
215 putAndWait(row1, famName, htable2, htable1);
216
217
218 assertEquals("Puts were replicated back ", 2, getCount(htable1, put));
219
220
221 deleteAndWait(row, htable1, htable2);
222
223
224 assertEquals("Puts were replicated back ", 2, getCount(htable2, put));
225
226 deleteAndWait(row1, htable2, htable1);
227
228 assertEquals("Deletes were replicated back ", 2, getCount(htable1, delete));
229 utility2.shutdownMiniCluster();
230 utility1.shutdownMiniCluster();
231 }
232
233 private int getCount(HTable t, byte[] type) throws IOException {
234 Get test = new Get(row);
235 test.setAttribute("count", new byte[]{});
236 Result res = t.get(test);
237 return Bytes.toInt(res.getValue(count, type));
238 }
239
240 private void deleteAndWait(byte[] row, HTable source, HTable target)
241 throws Exception {
242 Delete del = new Delete(row);
243 source.delete(del);
244
245 Get get = new Get(row);
246 for (int i = 0; i < NB_RETRIES; i++) {
247 if (i==NB_RETRIES-1) {
248 fail("Waited too much time for del replication");
249 }
250 Result res = target.get(get);
251 if (res.size() >= 1) {
252 LOG.info("Row not deleted");
253 Thread.sleep(SLEEP_TIME);
254 } else {
255 break;
256 }
257 }
258 }
259
260 private void check(byte[] row, byte[] fam, HTable t) throws IOException {
261 Get get = new Get(row);
262 Result res = t.get(get);
263 if (res.size() == 0) {
264 fail("Row is missing");
265 }
266 }
267
268 private void putAndWait(byte[] row, byte[] fam, HTable source, HTable target)
269 throws Exception {
270 Put put = new Put(row);
271 put.add(fam, row, row);
272 source.put(put);
273
274 Get get = new Get(row);
275 for (int i = 0; i < NB_RETRIES; i++) {
276 if (i==NB_RETRIES-1) {
277 fail("Waited too much time for put replication");
278 }
279 Result res = target.get(get);
280 if (res.size() == 0) {
281 LOG.info("Row not available");
282 Thread.sleep(SLEEP_TIME);
283 } else {
284 assertArrayEquals(res.value(), row);
285 break;
286 }
287 }
288 }
289
290
291
292
293
294
295 public static class CoprocessorCounter extends BaseRegionObserver {
296 private int nCount = 0;
297 private int nDelete = 0;
298
299 @Override
300 public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
301 final Put put, final WALEdit edit,
302 final boolean writeToWAL)
303 throws IOException {
304 nCount++;
305 }
306 @Override
307 public void postDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
308 final Delete delete, final WALEdit edit,
309 final boolean writeToWAL)
310 throws IOException {
311 nDelete++;
312 }
313 @Override
314 public void preGet(final ObserverContext<RegionCoprocessorEnvironment> c,
315 final Get get, final List<KeyValue> result) throws IOException {
316 if (get.getAttribute("count") != null) {
317 result.clear();
318
319 result.add(new KeyValue(count, count, delete, Bytes.toBytes(nDelete)));
320 result.add(new KeyValue(count, count, put, Bytes.toBytes(nCount)));
321 c.bypass();
322 }
323 }
324 }
325
326 @org.junit.Rule
327 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
328 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
329 }
330