1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.lang.reflect.InvocationTargetException;
30 import java.lang.reflect.Method;
31 import java.util.List;
32 import java.util.Map;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.client.Get;
38 import org.apache.hadoop.hbase.client.HBaseAdmin;
39 import org.apache.hadoop.hbase.client.HConnection;
40 import org.apache.hadoop.hbase.client.HConnectionManager;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Result;
44 import org.apache.hadoop.hbase.client.ResultScanner;
45 import org.apache.hadoop.hbase.client.Scan;
46 import org.apache.hadoop.hbase.master.HMaster;
47 import org.apache.hadoop.hbase.master.LoadBalancer;
48 import org.apache.hadoop.hbase.master.balancer.DefaultLoadBalancer;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.Threads;
51 import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
52 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
53 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
54 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
55 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
56 import org.apache.zookeeper.CreateMode;
57 import org.apache.zookeeper.KeeperException;
58 import org.apache.zookeeper.ZooDefs;
59 import org.apache.zookeeper.ZooKeeper;
60 import org.apache.zookeeper.ZooKeeper.States;
61 import org.apache.zookeeper.data.ACL;
62 import org.apache.zookeeper.data.Stat;
63 import org.junit.AfterClass;
64 import org.junit.Assert;
65 import org.junit.Before;
66 import org.junit.BeforeClass;
67 import org.junit.Test;
68 import org.junit.experimental.categories.Category;
69
70
71
72 @Category(LargeTests.class)
73 public class TestZooKeeper {
74 private final Log LOG = LogFactory.getLog(this.getClass());
75
76 private final static HBaseTestingUtility
77 TEST_UTIL = new HBaseTestingUtility();
78
79
80
81
82 @BeforeClass
83 public static void setUpBeforeClass() throws Exception {
84
85 Configuration conf = TEST_UTIL.getConfiguration();
86 TEST_UTIL.startMiniZKCluster();
87 conf.setBoolean("dfs.support.append", true);
88 conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
89 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
90 LoadBalancer.class);
91 TEST_UTIL.startMiniCluster(2);
92 }
93
94
95
96
97 @AfterClass
98 public static void tearDownAfterClass() throws Exception {
99 TEST_UTIL.shutdownMiniCluster();
100 }
101
102
103
104
105 @Before
106 public void setUp() throws Exception {
107 TEST_UTIL.ensureSomeRegionServersAvailable(2);
108 }
109
110
111 private ZooKeeperWatcher getZooKeeperWatcher(HConnection c)
112 throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
113 Method getterZK = c.getClass().getDeclaredMethod("getKeepAliveZooKeeperWatcher");
114 getterZK.setAccessible(true);
115 return (ZooKeeperWatcher) getterZK.invoke(c);
116 }
117
118
119
120
121
122
123
124
125
126 public void testClientSessionExpired() throws Exception {
127 Configuration c = new Configuration(TEST_UTIL.getConfiguration());
128
129
130 c.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "1111");
131
132 HConnection connection = HConnectionManager.getConnection(c);
133
134 ZooKeeperWatcher connectionZK = getZooKeeperWatcher(connection);
135 LOG.info("ZooKeeperWatcher= 0x"+ Integer.toHexString(
136 connectionZK.hashCode()));
137 LOG.info("getRecoverableZooKeeper= 0x"+ Integer.toHexString(
138 connectionZK.getRecoverableZooKeeper().hashCode()));
139 LOG.info("session="+Long.toHexString(
140 connectionZK.getRecoverableZooKeeper().getSessionId()));
141
142 TEST_UTIL.expireSession(connectionZK);
143
144 LOG.info("Before using zkw state=" +
145 connectionZK.getRecoverableZooKeeper().getState());
146
147 try {
148 connectionZK.getRecoverableZooKeeper().getZooKeeper().exists(
149 "/1/1", false);
150 } catch (KeeperException ignored) {
151 }
152
153
154 States state = connectionZK.getRecoverableZooKeeper().getState();
155 LOG.info("After using zkw state=" + state);
156 LOG.info("session="+Long.toHexString(
157 connectionZK.getRecoverableZooKeeper().getSessionId()));
158
159
160 final long limit1 = System.currentTimeMillis() + 3000;
161 while (System.currentTimeMillis() < limit1 && state != States.CLOSED){
162 state = connectionZK.getRecoverableZooKeeper().getState();
163 }
164 LOG.info("After using zkw loop=" + state);
165 LOG.info("ZooKeeper should have timed out");
166 LOG.info("session="+Long.toHexString(
167 connectionZK.getRecoverableZooKeeper().getSessionId()));
168
169
170
171
172
173
174
175 ZooKeeperWatcher newConnectionZK = getZooKeeperWatcher(connection);
176
177 States state2 = newConnectionZK.getRecoverableZooKeeper().getState();
178 LOG.info("After new get state=" +state2);
179
180
181
182 final long limit2 = System.currentTimeMillis() + 3000;
183 while (System.currentTimeMillis() < limit2 &&
184 state2 != States.CONNECTED && state2 != States.CONNECTING) {
185
186 newConnectionZK = getZooKeeperWatcher(connection);
187 state2 = newConnectionZK.getRecoverableZooKeeper().getState();
188 }
189 LOG.info("After new get state loop=" + state2);
190
191 Assert.assertTrue(
192 state2 == States.CONNECTED || state2 == States.CONNECTING);
193
194 connection.close();
195 }
196
197 @Test (timeout = 60000)
198 public void testRegionServerSessionExpired() throws Exception {
199 LOG.info("Starting testRegionServerSessionExpired");
200 int metaIndex = TEST_UTIL.getMiniHBaseCluster().getServerWithMeta();
201 TEST_UTIL.expireRegionServerSession(metaIndex);
202 testSanity("testRegionServerSessionExpired");
203 }
204
205
206
207
208 public void testMasterSessionExpired() throws Exception {
209 LOG.info("Starting testMasterSessionExpired");
210 TEST_UTIL.expireMasterSession();
211 testSanity("testMasterSessionExpired");
212 }
213
214
215
216
217
218
219 @Test(timeout = 60000)
220 public void testMasterZKSessionRecoveryFailure() throws Exception {
221 LOG.info("Starting testMasterZKSessionRecoveryFailure");
222 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
223 HMaster m = cluster.getMaster();
224 m.abort("Test recovery from zk session expired",
225 new KeeperException.SessionExpiredException());
226 assertFalse(m.isStopped());
227 testSanity("testMasterZKSessionRecoveryFailure");
228 }
229
230
231
232
233
234 private void testSanity(final String testName) throws Exception{
235 String tableName = testName + "_" + System.currentTimeMillis();
236 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
237 HColumnDescriptor family = new HColumnDescriptor("fam");
238 desc.addFamily(family);
239 LOG.info("Creating table " + tableName);
240 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
241 try {
242 admin.createTable(desc);
243 } finally {
244 admin.close();
245 }
246
247 HTable table =
248 new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
249 Put put = new Put(Bytes.toBytes("testrow"));
250 put.add(Bytes.toBytes("fam"),
251 Bytes.toBytes("col"), Bytes.toBytes("testdata"));
252 LOG.info("Putting table " + tableName);
253 table.put(put);
254 table.close();
255 }
256
257 @Test
258 public void testMultipleZK()
259 throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
260 HTable localMeta =
261 new HTable(new Configuration(TEST_UTIL.getConfiguration()), TableName.META_TABLE_NAME);
262 Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration());
263 otherConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
264 HTable ipMeta = new HTable(otherConf, TableName.META_TABLE_NAME);
265
266
267 final byte [] row = new byte [] {'r'};
268 localMeta.exists(new Get(row));
269 ipMeta.exists(new Get(row));
270
271
272 ZooKeeperWatcher z1 =
273 getZooKeeperWatcher(HConnectionManager.getConnection(localMeta.getConfiguration()));
274 ZooKeeperWatcher z2 =
275 getZooKeeperWatcher(HConnectionManager.getConnection(otherConf));
276 assertFalse(z1 == z2);
277 assertFalse(z1.getQuorum().equals(z2.getQuorum()));
278
279 localMeta.close();
280 ipMeta.close();
281 }
282
283
284
285
286
287 @Test
288 public void testCreateWithParents() throws Exception {
289 ZooKeeperWatcher zkw =
290 new ZooKeeperWatcher(new Configuration(TEST_UTIL.getConfiguration()),
291 TestZooKeeper.class.getName(), null);
292 byte[] expectedData = new byte[] { 1, 2, 3 };
293 ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4/testCreateWithParents", expectedData);
294 byte[] data = ZKUtil.getData(zkw, "/l1/l2/l3/l4/testCreateWithParents");
295 assertTrue(Bytes.equals(expectedData, data));
296 ZKUtil.deleteNodeRecursively(zkw, "/l1");
297
298 ZKUtil.createWithParents(zkw, "/testCreateWithParents", expectedData);
299 data = ZKUtil.getData(zkw, "/testCreateWithParents");
300 assertTrue(Bytes.equals(expectedData, data));
301 ZKUtil.deleteNodeRecursively(zkw, "/testCreateWithParents");
302 }
303
304
305
306
307
308
309 @Test
310 public void testZNodeDeletes() throws Exception {
311 ZooKeeperWatcher zkw = new ZooKeeperWatcher(
312 new Configuration(TEST_UTIL.getConfiguration()),
313 TestZooKeeper.class.getName(), null);
314 ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4");
315 try {
316 ZKUtil.deleteNode(zkw, "/l1/l2");
317 fail("We should not be able to delete if znode has childs");
318 } catch (KeeperException ex) {
319 assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
320 }
321 ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
322
323 assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
324
325
326 ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
327
328 ZKUtil.deleteNode(zkw, "/l1");
329 assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null));
330 }
331
332 @Test
333 public void testClusterKey() throws Exception {
334 testKey("server", "2181", "hbase");
335 testKey("server1,server2,server3", "2181", "hbase");
336 try {
337 ZKUtil.transformClusterKey("2181:hbase");
338 } catch (IOException ex) {
339
340 }
341 }
342
343 private void testKey(String ensemble, String port, String znode)
344 throws IOException {
345 Configuration conf = new Configuration();
346 String key = ensemble+":"+port+":"+znode;
347 String[] parts = ZKUtil.transformClusterKey(key);
348 assertEquals(ensemble, parts[0]);
349 assertEquals(port, parts[1]);
350 assertEquals(znode, parts[2]);
351 ZKUtil.applyClusterKeyToConf(conf, key);
352 assertEquals(parts[0], conf.get(HConstants.ZOOKEEPER_QUORUM));
353 assertEquals(parts[1], conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
354 assertEquals(parts[2], conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
355 String reconstructedKey = ZKUtil.getZooKeeperClusterKey(conf);
356 assertEquals(key, reconstructedKey);
357 }
358
359
360
361
362
363
364
365
366 @Test
367 public void testCreateSilentIsReallySilent() throws InterruptedException,
368 KeeperException, IOException {
369 Configuration c = TEST_UTIL.getConfiguration();
370
371 String aclZnode = "/aclRoot";
372 String quorumServers = ZKConfig.getZKQuorumServersString(c);
373 int sessionTimeout = 5 * 1000;
374 ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
375 zk.addAuthInfo("digest", "hbase:rox".getBytes());
376
377
378
379 ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
380 "testMasterAddressManagerFromZK", null);
381
382
383 Stat s = null;
384 List<ACL> oldACL = null;
385 while (true) {
386 try {
387 s = new Stat();
388 oldACL = zk.getACL("/", s);
389 break;
390 } catch (KeeperException e) {
391 switch (e.code()) {
392 case CONNECTIONLOSS:
393 case SESSIONEXPIRED:
394 case OPERATIONTIMEOUT:
395 LOG.warn("Possibly transient ZooKeeper exception", e);
396 Threads.sleep(100);
397 break;
398 default:
399 throw e;
400 }
401 }
402 }
403
404
405
406 while (true) {
407 try {
408 zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
409 break;
410 } catch (KeeperException e) {
411 switch (e.code()) {
412 case CONNECTIONLOSS:
413 case SESSIONEXPIRED:
414 case OPERATIONTIMEOUT:
415 LOG.warn("Possibly transient ZooKeeper exception: " + e);
416 Threads.sleep(100);
417 break;
418 default:
419 throw e;
420 }
421 }
422 }
423
424 while (true) {
425 try {
426 zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
427 break;
428 } catch (KeeperException e) {
429 switch (e.code()) {
430 case CONNECTIONLOSS:
431 case SESSIONEXPIRED:
432 case OPERATIONTIMEOUT:
433 LOG.warn("Possibly transient ZooKeeper exception: " + e);
434 Threads.sleep(100);
435 break;
436 default:
437 throw e;
438 }
439 }
440 }
441 zk.close();
442 ZKUtil.createAndFailSilent(zk2, aclZnode);
443
444
445 ZooKeeper zk3 = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
446 zk3.addAuthInfo("digest", "hbase:rox".getBytes());
447 try {
448 zk3.setACL("/", oldACL, -1);
449 } finally {
450 zk3.close();
451 }
452 }
453
454
455
456
457
458 @Test
459 @SuppressWarnings("deprecation")
460 public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
461 throws Exception {
462 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
463 "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
464 ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
465 }
466
467
468
469
470
471
472 @Test
473 public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
474 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
475 cluster.startRegionServer();
476 HMaster m = cluster.getMaster();
477 ZooKeeperWatcher zkw = m.getZooKeeperWatcher();
478 int expectedNumOfListeners = zkw.getNumberOfListeners();
479
480 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
481 try {
482 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
483 Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
484 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
485 String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry";
486 admin.createTable(new HTableDescriptor(TableName.valueOf(tableName)), SPLIT_KEYS);
487 ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
488 ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
489 m.getZooKeeperWatcher().close();
490 MockLoadBalancer.retainAssignCalled = false;
491 m.abort("Test recovery from zk session expired",
492 new KeeperException.SessionExpiredException());
493 assertFalse(m.isStopped());
494
495
496 assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
497
498 assertEquals(expectedNumOfListeners, zkw.getNumberOfListeners());
499 } finally {
500 admin.close();
501 }
502 }
503
504
505
506
507
508 @Test(timeout = 240000)
509 public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException,
510 KeeperException, InterruptedException {
511 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
512 cluster.startRegionServer();
513 HMaster m = cluster.getMaster();
514
515 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
516 HTable table = null;
517 try {
518 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
519 Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
520
521 String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry";
522 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
523 HColumnDescriptor hcd = new HColumnDescriptor("col");
524 htd.addFamily(hcd);
525 admin.createTable(htd, SPLIT_KEYS);
526 ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
527 ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
528 table = new HTable(TEST_UTIL.getConfiguration(), tableName);
529 Put p;
530 int numberOfPuts;
531 for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
532 p = new Put(Bytes.toBytes(numberOfPuts));
533 p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts));
534 table.put(p);
535 }
536 m.getZooKeeperWatcher().close();
537 m.abort("Test recovery from zk session expired",
538 new KeeperException.SessionExpiredException());
539 assertFalse(m.isStopped());
540 cluster.getRegionServer(0).abort("Aborting");
541
542
543 Scan scan = new Scan();
544 int numberOfRows = 0;
545 ResultScanner scanner = table.getScanner(scan);
546 Result[] result = scanner.next(1);
547 while (result != null && result.length > 0) {
548 numberOfRows++;
549 result = scanner.next(1);
550 }
551 assertEquals("Number of rows should be equal to number of puts.", numberOfPuts,
552 numberOfRows);
553 } finally {
554 if (table != null) table.close();
555 admin.close();
556 }
557 }
558
559 static class MockLoadBalancer extends DefaultLoadBalancer {
560 static boolean retainAssignCalled = false;
561
562 @Override
563 public Map<ServerName, List<HRegionInfo>> retainAssignment(
564 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
565 retainAssignCalled = true;
566 return super.retainAssignment(regions, servers);
567 }
568 }
569
570 }
571