1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertNotNull;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.HashSet;
30 import java.util.Iterator;
31 import java.util.List;
32 import java.util.Set;
33 import java.util.TreeSet;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.Abortable;
41 import org.apache.hadoop.hbase.ClusterStatus;
42 import org.apache.hadoop.hbase.HBaseConfiguration;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.HRegionInfo;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.LargeTests;
49 import org.apache.hadoop.hbase.MasterNotRunningException;
50 import org.apache.hadoop.hbase.MiniHBaseCluster;
51 import org.apache.hadoop.hbase.ServerName;
52 import org.apache.hadoop.hbase.client.HTable;
53 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
54 import org.apache.hadoop.hbase.executor.RegionTransitionData;
55 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
56 import org.apache.hadoop.hbase.regionserver.HRegion;
57 import org.apache.hadoop.hbase.regionserver.HRegionServer;
58 import org.apache.hadoop.hbase.util.Bytes;
59 import org.apache.hadoop.hbase.util.FSTableDescriptors;
60 import org.apache.hadoop.hbase.util.JVMClusterUtil;
61 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
62 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
63 import org.apache.hadoop.hbase.util.Threads;
64 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
65 import org.apache.hadoop.hbase.zookeeper.ZKTable;
66 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
67 import org.junit.Test;
68 import org.junit.experimental.categories.Category;
69
70 @Category(LargeTests.class)
71 public class TestMasterFailover {
72 private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
73
74 @Test (timeout=180000)
75 public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
76 throws Exception {
77 LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
78 final int NUM_MASTERS = 1;
79 final int NUM_RS = 2;
80
81 Configuration conf = HBaseConfiguration.create();
82 conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
83 conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
84
85 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
86
87 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
88 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
89
90
91 List<RegionServerThread> regionServerThreads =
92 cluster.getRegionServerThreads();
93 int count = -1;
94 HRegion metaRegion = null;
95 for (RegionServerThread regionServerThread : regionServerThreads) {
96 HRegionServer regionServer = regionServerThread.getRegionServer();
97 metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
98 count++;
99 regionServer.abort("");
100 if (null != metaRegion) break;
101 }
102 HRegionServer regionServer = cluster.getRegionServer(count);
103
104 TEST_UTIL.shutdownMiniHBaseCluster();
105
106
107 ZooKeeperWatcher zkw =
108 HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL,
109 metaRegion, regionServer.getServerName());
110
111 LOG.info("Staring cluster for second time");
112 TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
113
114
115 log("Waiting for no more RIT");
116 ZKAssign.blockUntilNoRIT(zkw);
117
118 zkw.close();
119
120 TEST_UTIL.shutdownMiniCluster();
121 }
122
123
124
125
126
127
128
129
130
131 @Test (timeout=240000)
132 public void testSimpleMasterFailover() throws Exception {
133
134 final int NUM_MASTERS = 3;
135 final int NUM_RS = 3;
136
137
138 Configuration conf = HBaseConfiguration.create();
139
140
141 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
142 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
143 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
144
145
146 List<MasterThread> masterThreads = cluster.getMasterThreads();
147
148
149 for (MasterThread mt : masterThreads) {
150 assertTrue(mt.isAlive());
151 }
152
153
154 int numActive = 0;
155 int activeIndex = -1;
156 ServerName activeName = null;
157 HMaster active = null;
158 for (int i = 0; i < masterThreads.size(); i++) {
159 if (masterThreads.get(i).getMaster().isActiveMaster()) {
160 numActive++;
161 activeIndex = i;
162 active = masterThreads.get(activeIndex).getMaster();
163 activeName = active.getServerName();
164 }
165 }
166 assertEquals(1, numActive);
167 assertEquals(NUM_MASTERS, masterThreads.size());
168 LOG.info("Active master " + activeName);
169
170
171 assertNotNull(active);
172 ClusterStatus status = active.getClusterStatus();
173 assertTrue(status.getMaster().equals(activeName));
174 assertEquals(2, status.getBackupMastersSize());
175 assertEquals(2, status.getBackupMasters().size());
176
177
178 int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
179 HMaster master = cluster.getMaster(backupIndex);
180 LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
181 cluster.stopMaster(backupIndex, false);
182 cluster.waitOnMaster(backupIndex);
183
184
185 for (int i = 0; i < masterThreads.size(); i++) {
186 if (masterThreads.get(i).getMaster().isActiveMaster()) {
187 assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
188 activeIndex = i;
189 active = masterThreads.get(activeIndex).getMaster();
190 }
191 }
192 assertEquals(1, numActive);
193 assertEquals(2, masterThreads.size());
194 int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
195 LOG.info("Active master " + active.getServerName() + " managing " + rsCount + " regions servers");
196 assertEquals(3, rsCount);
197
198
199 assertNotNull(active);
200 status = active.getClusterStatus();
201 assertTrue(status.getMaster().equals(activeName));
202 assertEquals(1, status.getBackupMastersSize());
203 assertEquals(1, status.getBackupMasters().size());
204
205
206 LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
207 cluster.stopMaster(activeIndex, false);
208 cluster.waitOnMaster(activeIndex);
209
210
211 assertTrue(cluster.waitForActiveAndReadyMaster());
212
213 LOG.debug("\n\nVerifying backup master is now active\n");
214
215 assertEquals(1, masterThreads.size());
216
217
218 active = masterThreads.get(0).getMaster();
219 assertNotNull(active);
220 status = active.getClusterStatus();
221 ServerName mastername = status.getMaster();
222 assertTrue(mastername.equals(active.getServerName()));
223 assertTrue(active.isActiveMaster());
224 assertEquals(0, status.getBackupMastersSize());
225 assertEquals(0, status.getBackupMasters().size());
226 int rss = status.getServersSize();
227 LOG.info("Active master " + mastername.getServerName() + " managing " +
228 rss + " region servers");
229 assertEquals(3, rss);
230
231
232 TEST_UTIL.shutdownMiniCluster();
233 }
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314 @Test (timeout=180000)
315 public void testMasterFailoverWithMockedRIT() throws Exception {
316
317 final int NUM_MASTERS = 1;
318 final int NUM_RS = 3;
319
320
321 Configuration conf = HBaseConfiguration.create();
322
323 conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
324 conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000);
325 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 3);
326 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 3);
327
328
329 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
330 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
331 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
332 log("Cluster started");
333
334
335 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
336
337
338 List<MasterThread> masterThreads = cluster.getMasterThreads();
339 assertEquals(1, masterThreads.size());
340
341
342 assertTrue(cluster.waitForActiveAndReadyMaster());
343 HMaster master = masterThreads.get(0).getMaster();
344 assertTrue(master.isActiveMaster());
345 assertTrue(master.isInitialized());
346
347
348 master.balanceSwitch(false);
349
350
351 byte [] FAMILY = Bytes.toBytes("family");
352 byte [][] SPLIT_KEYS = new byte [][] {
353 new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
354 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
355 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
356 Bytes.toBytes("iii"), Bytes.toBytes("jjj")
357 };
358
359 byte [] enabledTable = Bytes.toBytes("enabledTable");
360 HTableDescriptor htdEnabled = new HTableDescriptor(enabledTable);
361 htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
362
363 FileSystem filesystem = FileSystem.get(conf);
364 Path rootdir = filesystem.makeQualified(
365 new Path(conf.get(HConstants.HBASE_DIR)));
366
367 FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled);
368
369 HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(), null, null);
370 createRegion(hriEnabled, rootdir, conf, htdEnabled);
371
372 List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
373 TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
374
375 byte [] disabledTable = Bytes.toBytes("disabledTable");
376 HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
377 htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
378
379 FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
380 HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
381 createRegion(hriDisabled, rootdir, conf, htdDisabled);
382 List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
383 TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
384
385 log("Regions in META have been created");
386
387
388 assertEquals(2, cluster.countServedRegions());
389
390
391 HRegionServer hrs = cluster.getRegionServer(0);
392 ServerName serverName = hrs.getServerName();
393 HRegionInfo closingRegion = enabledRegions.remove(0);
394
395 List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
396 enabledAndAssignedRegions.add(enabledRegions.remove(0));
397 enabledAndAssignedRegions.add(enabledRegions.remove(0));
398 enabledAndAssignedRegions.add(closingRegion);
399
400 List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
401 disabledAndAssignedRegions.add(disabledRegions.remove(0));
402 disabledAndAssignedRegions.add(disabledRegions.remove(0));
403
404
405 for (HRegionInfo hri : enabledAndAssignedRegions) {
406 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
407 new RegionPlan(hri, null, serverName));
408 master.assignRegion(hri);
409 }
410 for (HRegionInfo hri : disabledAndAssignedRegions) {
411 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
412 new RegionPlan(hri, null, serverName));
413 master.assignRegion(hri);
414 }
415
416
417 log("Waiting for assignment to finish");
418 ZKAssign.blockUntilNoRIT(zkw);
419 log("Assignment completed");
420
421
422 log("Aborting master");
423 cluster.abortMaster(0);
424 cluster.waitOnMaster(0);
425 log("Master has aborted");
426
427
428
429
430
431
432 List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
433 List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
434
435 log("Beginning to mock scenarios");
436
437
438 ZKTable zktable = new ZKTable(zkw);
439 zktable.setDisabledTable(Bytes.toString(disabledTable));
440
441
442
443
444
445
446 HRegionInfo region = enabledRegions.remove(0);
447 regionsThatShouldBeOnline.add(region);
448 ZKAssign.createNodeOffline(zkw, region, serverName);
449
450
451
452
453 regionsThatShouldBeOnline.add(closingRegion);
454 ZKAssign.createNodeClosing(zkw, closingRegion, serverName);
455
456
457
458
459
460
461 region = enabledRegions.remove(0);
462 regionsThatShouldBeOnline.add(region);
463 int version = ZKAssign.createNodeClosing(zkw, region, serverName);
464 ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
465
466
467 region = disabledRegions.remove(0);
468 regionsThatShouldBeOffline.add(region);
469 version = ZKAssign.createNodeClosing(zkw, region, serverName);
470 ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
471
472
473
474
475
476
477 region = enabledRegions.remove(0);
478 regionsThatShouldBeOnline.add(region);
479 ZKAssign.createNodeOffline(zkw, region, serverName);
480 ZKAssign.transitionNodeOpening(zkw, region, serverName);
481
482
483
484
485
486
487 region = enabledRegions.remove(0);
488 regionsThatShouldBeOnline.add(region);
489 ZKAssign.createNodeOffline(zkw, region, serverName);
490 hrs.openRegion(region);
491 while (true) {
492 RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
493 if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
494 break;
495 }
496 Thread.sleep(100);
497 }
498
499
500 region = disabledRegions.remove(0);
501 regionsThatShouldBeOffline.add(region);
502 ZKAssign.createNodeOffline(zkw, region, serverName);
503 hrs.openRegion(region);
504 while (true) {
505 RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
506 if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
507 break;
508 }
509 Thread.sleep(100);
510 }
511
512
513
514
515
516
517
518
519
520 log("Done mocking data up in ZK");
521
522
523 log("Starting up a new master");
524 master = cluster.startMaster().getMaster();
525 log("Waiting for master to be ready");
526 cluster.waitForActiveAndReadyMaster();
527 log("Master is ready");
528
529
530 log("Waiting for no more RIT");
531 ZKAssign.blockUntilNoRIT(zkw);
532 log("No more RIT in ZK, now doing final test verification");
533
534
535 Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
536 for (JVMClusterUtil.RegionServerThread rst :
537 cluster.getRegionServerThreads()) {
538 onlineRegions.addAll(rst.getRegionServer().getOnlineRegions());
539 }
540
541
542 for (HRegionInfo hri : regionsThatShouldBeOnline) {
543 assertTrue(onlineRegions.contains(hri));
544 }
545
546
547 for (HRegionInfo hri : regionsThatShouldBeOffline) {
548 assertFalse(onlineRegions.contains(hri));
549 }
550
551 log("Done with verification, all passed, shutting down cluster");
552
553
554 TEST_UTIL.shutdownMiniCluster();
555 }
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614 @Test(timeout = 180000)
615 public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
616
617 final int NUM_MASTERS = 1;
618 final int NUM_RS = 2;
619
620
621 Configuration conf = HBaseConfiguration.create();
622
623 conf.setInt("hbase.master.assignment.timeoutmonitor.period", 4000);
624 conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
625 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
626 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
627
628
629 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
630 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
631 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
632 log("Cluster started");
633
634
635 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
636 "unittest", new Abortable() {
637
638 @Override
639 public void abort(String why, Throwable e) {
640 LOG.error("Fatal ZK Error: " + why, e);
641 org.junit.Assert.assertFalse("Fatal ZK error", true);
642 }
643
644 @Override
645 public boolean isAborted() {
646 return false;
647 }
648
649 });
650
651
652 List<MasterThread> masterThreads = cluster.getMasterThreads();
653 assertEquals(1, masterThreads.size());
654
655
656 assertTrue(cluster.waitForActiveAndReadyMaster());
657 HMaster master = masterThreads.get(0).getMaster();
658 assertTrue(master.isActiveMaster());
659 assertTrue(master.isInitialized());
660
661
662 master.balanceSwitch(false);
663
664
665 byte [] FAMILY = Bytes.toBytes("family");
666 byte[][] SPLIT_KEYS =
667 TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 30);
668
669 byte [] enabledTable = Bytes.toBytes("enabledTable");
670 HTableDescriptor htdEnabled = new HTableDescriptor(enabledTable);
671 htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
672 FileSystem filesystem = FileSystem.get(conf);
673 Path rootdir = filesystem.makeQualified(
674 new Path(conf.get(HConstants.HBASE_DIR)));
675
676 FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdEnabled);
677 HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getName(),
678 null, null);
679 createRegion(hriEnabled, rootdir, conf, htdEnabled);
680
681 List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
682 TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
683
684 byte [] disabledTable = Bytes.toBytes("disabledTable");
685 HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
686 htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
687
688 FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
689 HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
690 createRegion(hriDisabled, rootdir, conf, htdDisabled);
691
692 List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
693 TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
694
695 log("Regions in META have been created");
696
697
698 assertEquals(2, cluster.countServedRegions());
699
700
701 List<RegionServerThread> regionservers =
702 cluster.getRegionServerThreads();
703 HRegionServer hrs = regionservers.get(0).getRegionServer();
704
705
706 RegionServerThread hrsDeadThread = regionservers.get(1);
707 HRegionServer hrsDead = hrsDeadThread.getRegionServer();
708 ServerName deadServerName = hrsDead.getServerName();
709
710
711 List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
712 enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6));
713 enabledRegions.removeAll(enabledAndAssignedRegions);
714 List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
715 disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6));
716 disabledRegions.removeAll(disabledAndAssignedRegions);
717
718
719 for (HRegionInfo hri : enabledAndAssignedRegions) {
720 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
721 new RegionPlan(hri, null, hrs.getServerName()));
722 master.assignRegion(hri);
723 }
724 for (HRegionInfo hri : disabledAndAssignedRegions) {
725 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
726 new RegionPlan(hri, null, hrs.getServerName()));
727 master.assignRegion(hri);
728 }
729
730 log("Waiting for assignment to finish");
731 ZKAssign.blockUntilNoRIT(zkw);
732 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
733 log("Assignment completed");
734
735 assertTrue(" Table must be enabled.", master.getAssignmentManager()
736 .getZKTable().isEnabledTable("enabledTable"));
737
738 List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
739 enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6));
740 enabledRegions.removeAll(enabledAndOnDeadRegions);
741 List<HRegionInfo> disabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
742 disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6));
743 disabledRegions.removeAll(disabledAndOnDeadRegions);
744
745
746 for (HRegionInfo hri : enabledAndOnDeadRegions) {
747 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
748 new RegionPlan(hri, null, deadServerName));
749 master.assignRegion(hri);
750 }
751 for (HRegionInfo hri : disabledAndOnDeadRegions) {
752 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
753 new RegionPlan(hri, null, deadServerName));
754 master.assignRegion(hri);
755 }
756
757
758 log("Waiting for assignment to finish");
759 ZKAssign.blockUntilNoRIT(zkw);
760 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
761 log("Assignment completed");
762
763
764
765 verifyRegionLocation(hrs, enabledAndAssignedRegions);
766 verifyRegionLocation(hrs, disabledAndAssignedRegions);
767 verifyRegionLocation(hrsDead, enabledAndOnDeadRegions);
768 verifyRegionLocation(hrsDead, disabledAndOnDeadRegions);
769
770 assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
771 enabledAndAssignedRegions.size() >= 2);
772 assertTrue(" Didn't get enough regions of disalbedTable on live rs.",
773 disabledAndAssignedRegions.size() >= 2);
774 assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.",
775 enabledAndOnDeadRegions.size() >= 2);
776 assertTrue(" Didn't get enough regions of disalbedTable on dead rs.",
777 disabledAndOnDeadRegions.size() >= 2);
778
779
780 log("Aborting master");
781 cluster.abortMaster(0);
782 cluster.waitOnMaster(0);
783 log("Master has aborted");
784
785
786
787
788
789
790 List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
791 List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
792
793 log("Beginning to mock scenarios");
794
795
796 ZKTable zktable = new ZKTable(zkw);
797 zktable.setDisabledTable(Bytes.toString(disabledTable));
798
799 assertTrue(" The enabled table should be identified on master fail over.",
800 zktable.isEnabledTable("enabledTable"));
801
802
803
804
805
806
807 HRegionInfo region = enabledAndOnDeadRegions.remove(0);
808 regionsThatShouldBeOnline.add(region);
809 ZKAssign.createNodeClosing(zkw, region, deadServerName);
810 LOG.debug("\n\nRegion of enabled table was CLOSING on dead RS\n" +
811 region + "\n\n");
812
813
814 region = disabledAndOnDeadRegions.remove(0);
815 regionsThatShouldBeOffline.add(region);
816 ZKAssign.createNodeClosing(zkw, region, deadServerName);
817 LOG.debug("\n\nRegion of disabled table was CLOSING on dead RS\n" +
818 region + "\n\n");
819
820
821
822
823
824
825 region = enabledAndOnDeadRegions.remove(0);
826 regionsThatShouldBeOnline.add(region);
827 int version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
828 ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
829 LOG.debug("\n\nRegion of enabled table was CLOSED on dead RS\n" +
830 region + "\n\n");
831
832
833 region = disabledAndOnDeadRegions.remove(0);
834 regionsThatShouldBeOffline.add(region);
835 version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
836 ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
837 LOG.debug("\n\nRegion of disabled table was CLOSED on dead RS\n" +
838 region + "\n\n");
839
840
841
842
843
844
845 region = enabledRegions.remove(0);
846 regionsThatShouldBeOnline.add(region);
847 ZKAssign.createNodeOffline(zkw, region, deadServerName);
848 ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
849 LOG.debug("\n\nRegion of enabled table was OPENING on dead RS\n" +
850 region + "\n\n");
851
852
853 region = disabledRegions.remove(0);
854 regionsThatShouldBeOffline.add(region);
855 ZKAssign.createNodeOffline(zkw, region, deadServerName);
856 ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
857 LOG.debug("\n\nRegion of disabled table was OPENING on dead RS\n" +
858 region + "\n\n");
859
860
861
862
863
864
865 region = enabledRegions.remove(0);
866 regionsThatShouldBeOnline.add(region);
867 ZKAssign.createNodeOffline(zkw, region, deadServerName);
868 hrsDead.openRegion(region);
869 while (true) {
870 RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
871 if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
872 break;
873 }
874 Thread.sleep(100);
875 }
876 LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" +
877 region + "\n\n");
878
879
880 region = disabledRegions.remove(0);
881 regionsThatShouldBeOffline.add(region);
882 ZKAssign.createNodeOffline(zkw, region, deadServerName);
883 hrsDead.openRegion(region);
884 while (true) {
885 RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
886 if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
887 break;
888 }
889 Thread.sleep(100);
890 }
891 LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" +
892 region + "\n\n");
893
894
895
896
897
898
899 region = enabledRegions.remove(0);
900 regionsThatShouldBeOnline.add(region);
901 ZKAssign.createNodeOffline(zkw, region, deadServerName);
902 hrsDead.openRegion(region);
903 while (true) {
904 RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
905 if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
906 ZKAssign.deleteOpenedNode(zkw, region.getEncodedName());
907 break;
908 }
909 Thread.sleep(100);
910 }
911 LOG.debug("\n\nRegion of enabled table was open at steady-state on dead RS"
912 + "\n" + region + "\n\n");
913
914
915 region = disabledRegions.remove(0);
916 regionsThatShouldBeOffline.add(region);
917 ZKAssign.createNodeOffline(zkw, region, deadServerName);
918 hrsDead.openRegion(region);
919 while (true) {
920 RegionTransitionData rtd = ZKAssign.getData(zkw, region.getEncodedName());
921 if (rtd != null && rtd.getEventType() == EventType.RS_ZK_REGION_OPENED) {
922 ZKAssign.deleteOpenedNode(zkw, region.getEncodedName());
923 break;
924 }
925 Thread.sleep(100);
926 }
927 LOG.debug("\n\nRegion of disabled table was open at steady-state on dead RS"
928 + "\n" + region + "\n\n");
929
930
931
932
933
934 log("Done mocking data up in ZK");
935
936
937 log("Killing RS " + deadServerName);
938 hrsDead.abort("Killing for unit test");
939 log("RS " + deadServerName + " killed");
940
941
942
943 while (hrsDeadThread.isAlive()) {
944 Threads.sleep(10);
945 }
946 log("Starting up a new master");
947 master = cluster.startMaster().getMaster();
948 log("Waiting for master to be ready");
949 assertTrue(cluster.waitForActiveAndReadyMaster());
950 log("Master is ready");
951
952
953
954
955
956
957
958
959 region = enabledRegions.remove(0);
960 regionsThatShouldBeOnline.add(region);
961 master.assignmentManager.regionsInTransition.put(region.getEncodedName(),
962 new RegionState(region, RegionState.State.PENDING_OPEN, 0, null));
963 ZKAssign.createNodeOffline(zkw, region, master.getServerName());
964
965 region = disabledRegions.remove(0);
966 regionsThatShouldBeOffline.add(region);
967 master.assignmentManager.regionsInTransition.put(region.getEncodedName(),
968 new RegionState(region, RegionState.State.PENDING_OPEN, 0, null));
969 ZKAssign.createNodeOffline(zkw, region, master.getServerName());
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992 log("Waiting for no more RIT");
993 ZKAssign.blockUntilNoRIT(zkw);
994 log("No more RIT in ZK");
995 long now = System.currentTimeMillis();
996 final long maxTime = 120000;
997 boolean done = master.assignmentManager.waitUntilNoRegionsInTransition(maxTime);
998 if (!done) {
999 LOG.info("rit=" + master.assignmentManager.getRegionsInTransition());
1000 }
1001 long elapsed = System.currentTimeMillis() - now;
1002 assertTrue("Elapsed=" + elapsed + ", maxTime=" + maxTime + ", done=" + done,
1003 elapsed < maxTime);
1004 log("No more RIT in RIT map, doing final test verification");
1005
1006
1007 Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
1008 for (JVMClusterUtil.RegionServerThread rst :
1009 cluster.getRegionServerThreads()) {
1010 try {
1011 onlineRegions.addAll(rst.getRegionServer().getOnlineRegions());
1012 } catch (org.apache.hadoop.hbase.regionserver.RegionServerStoppedException e) {
1013 LOG.info("Got RegionServerStoppedException", e);
1014 }
1015 }
1016
1017
1018 for (HRegionInfo hri : regionsThatShouldBeOnline) {
1019 assertTrue("region=" + hri.getRegionNameAsString(), onlineRegions.contains(hri));
1020 }
1021
1022
1023 for (HRegionInfo hri : regionsThatShouldBeOffline) {
1024 assertFalse(onlineRegions.contains(hri));
1025 }
1026
1027 log("Done with verification, all passed, shutting down cluster");
1028
1029
1030 TEST_UTIL.shutdownMiniCluster();
1031 }
1032
1033 @Test(timeout = 180000)
1034 public void testRSKilledWithMockedOpeningRITGoingToDeadRS() throws Exception {
1035 final int NUM_MASTERS = 1;
1036 final int NUM_RS = 2;
1037
1038
1039 Configuration conf = HBaseConfiguration.create();
1040
1041 conf.setInt("hbase.master.assignment.timeoutmonitor.period", 10000);
1042 conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 30000);
1043 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
1044 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
1045
1046
1047 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
1048 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1049 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1050 log("Cluster started");
1051
1052
1053 ZooKeeperWatcher zkw =
1054 new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "unittest", new Abortable() {
1055
1056 @Override
1057 public void abort(String why, Throwable e) {
1058 LOG.error("Fatal ZK Error: " + why, e);
1059 org.junit.Assert.assertFalse("Fatal ZK error", true);
1060 }
1061
1062 @Override
1063 public boolean isAborted() {
1064 return false;
1065 }
1066
1067 });
1068
1069
1070 List<MasterThread> masterThreads = cluster.getMasterThreads();
1071 assertEquals(1, masterThreads.size());
1072
1073
1074 assertTrue(cluster.waitForActiveAndReadyMaster());
1075 HMaster master = masterThreads.get(0).getMaster();
1076 assertTrue(master.isActiveMaster());
1077 assertTrue(master.isInitialized());
1078
1079
1080 master.balanceSwitch(false);
1081
1082
1083 byte[] FAMILY = Bytes.toBytes("family");
1084 byte[][] SPLIT_KEYS =
1085 TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 15);
1086
1087 FileSystem filesystem = FileSystem.get(conf);
1088 Path rootdir = filesystem.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
1089
1090 byte[] disabledTable = Bytes.toBytes("disabledTable");
1091 HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
1092 htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
1093
1094 FSTableDescriptors.createTableDescriptor(filesystem, rootdir, htdDisabled);
1095 HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getName(), null, null);
1096 createRegion(hriDisabled, rootdir, conf, htdDisabled);
1097
1098 List<HRegionInfo> tableRegions =
1099 TEST_UTIL.createMultiRegionsInMeta(TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
1100
1101 log("Regions in META have been created");
1102
1103
1104 assertEquals(2, cluster.countServedRegions());
1105
1106
1107 List<RegionServerThread> regionservers = cluster.getRegionServerThreads();
1108 HRegionServer hrs = regionservers.get(0).getRegionServer();
1109
1110
1111 RegionServerThread hrsDeadThread = regionservers.get(1);
1112 HRegionServer hrsDead = hrsDeadThread.getRegionServer();
1113 ServerName deadServerName = hrsDead.getServerName();
1114
1115
1116 List<HRegionInfo> assignedRegionsOnLiveRS = new ArrayList<HRegionInfo>();
1117 assignedRegionsOnLiveRS.addAll(tableRegions.subList(0, 3));
1118 tableRegions.removeAll(assignedRegionsOnLiveRS);
1119
1120
1121 for (HRegionInfo hri : assignedRegionsOnLiveRS) {
1122 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
1123 new RegionPlan(hri, null, hrs.getServerName()));
1124 master.assignRegion(hri);
1125 }
1126
1127 log("Waiting for assignment to finish");
1128 ZKAssign.blockUntilNoRIT(zkw);
1129 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
1130 log("Assignment completed");
1131
1132
1133
1134 verifyRegionLocation(hrs, assignedRegionsOnLiveRS);
1135
1136 assertTrue(" Table must be enabled.", master.getAssignmentManager().getZKTable()
1137 .isEnabledTable("disabledTable"));
1138
1139 assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
1140 assignedRegionsOnLiveRS.size() >= 1);
1141
1142
1143 ZKTable zktable = master.assignmentManager.getZKTable();
1144 zktable.setDisablingTable("disabledTable");
1145
1146
1147 HRegionInfo region = assignedRegionsOnLiveRS.remove(0);
1148 master.assignmentManager.regionOffline(region);
1149 master.assignmentManager.regionsInTransition.put(region.getEncodedName(), new RegionState(
1150 region, RegionState.State.OPENING, System.currentTimeMillis(), deadServerName));
1151 ZKAssign.createNodeOffline(zkw, region, deadServerName);
1152 ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
1153
1154
1155 log("Killing RS " + deadServerName);
1156 hrsDead.abort("Killing for unit test");
1157 while (hrsDeadThread.isAlive()) {
1158 Threads.sleep(10);
1159 }
1160 log("RS " + deadServerName + " killed");
1161
1162 log("Waiting for no more RIT");
1163 ZKAssign.blockUntilNoRIT(zkw);
1164 log("No more RIT in ZK");
1165 assertTrue(master.assignmentManager.waitUntilNoRegionsInTransition(120000));
1166 }
1167
1168
1169
1170
1171 private void verifyRegionLocation(HRegionServer hrs, List<HRegionInfo> regions)
1172 throws IOException {
1173 List<HRegionInfo> tmpOnlineRegions = hrs.getOnlineRegions();
1174 Iterator<HRegionInfo> itr = regions.iterator();
1175 while (itr.hasNext()) {
1176 HRegionInfo tmp = itr.next();
1177 if (!tmpOnlineRegions.contains(tmp)) {
1178 itr.remove();
1179 }
1180 }
1181 }
1182
1183 HRegion createRegion(final HRegionInfo hri, final Path rootdir, final Configuration c,
1184 final HTableDescriptor htd)
1185 throws IOException {
1186 HRegion r = HRegion.createHRegion(hri, rootdir, c, htd);
1187
1188
1189
1190
1191
1192 HRegion.closeHRegion(r);
1193 return r;
1194 }
1195
1196
1197
1198
1199 private void log(String string) {
1200 LOG.info("\n\n" + string + " \n\n");
1201 }
1202
1203 @org.junit.Rule
1204 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1205 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1206 }
1207