1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.Set;
31 import java.util.TreeSet;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.Abortable;
39 import org.apache.hadoop.hbase.ClusterStatus;
40 import org.apache.hadoop.hbase.HBaseConfiguration;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.HColumnDescriptor;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HRegionInfo;
45 import org.apache.hadoop.hbase.HTableDescriptor;
46 import org.apache.hadoop.hbase.LargeTests;
47 import org.apache.hadoop.hbase.MiniHBaseCluster;
48 import org.apache.hadoop.hbase.RegionTransition;
49 import org.apache.hadoop.hbase.ServerName;
50 import org.apache.hadoop.hbase.TableName;
51 import org.apache.hadoop.hbase.catalog.MetaEditor;
52 import org.apache.hadoop.hbase.client.HTable;
53 import org.apache.hadoop.hbase.executor.EventType;
54 import org.apache.hadoop.hbase.master.RegionState.State;
55 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
56 import org.apache.hadoop.hbase.regionserver.HRegion;
57 import org.apache.hadoop.hbase.regionserver.HRegionServer;
58 import org.apache.hadoop.hbase.regionserver.RegionMergeTransaction;
59 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
60 import org.apache.hadoop.hbase.util.Bytes;
61 import org.apache.hadoop.hbase.util.FSTableDescriptors;
62 import org.apache.hadoop.hbase.util.FSUtils;
63 import org.apache.hadoop.hbase.util.JVMClusterUtil;
64 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
65 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
66 import org.apache.hadoop.hbase.util.Threads;
67 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
68 import org.apache.hadoop.hbase.zookeeper.ZKTable;
69 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
70 import org.apache.zookeeper.data.Stat;
71 import org.junit.Test;
72 import org.junit.experimental.categories.Category;
73
74 @Category(LargeTests.class)
75 public class TestMasterFailover {
76 private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157 @Test (timeout=240000)
158 public void testMasterFailoverWithMockedRIT() throws Exception {
159
160 final int NUM_MASTERS = 1;
161 final int NUM_RS = 3;
162
163
164 Configuration conf = HBaseConfiguration.create();
165 conf.setBoolean("hbase.assignment.usezk", true);
166
167
168 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
169 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
170 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
171 log("Cluster started");
172
173
174 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
175
176
177 List<MasterThread> masterThreads = cluster.getMasterThreads();
178 assertEquals(1, masterThreads.size());
179
180
181 assertTrue(cluster.waitForActiveAndReadyMaster());
182 HMaster master = masterThreads.get(0).getMaster();
183 assertTrue(master.isActiveMaster());
184 assertTrue(master.isInitialized());
185
186
187 master.balanceSwitch(false);
188
189
190 byte [] FAMILY = Bytes.toBytes("family");
191 byte [][] SPLIT_KEYS = new byte [][] {
192 new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
193 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
194 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
195 Bytes.toBytes("iii"), Bytes.toBytes("jjj")
196 };
197
198 byte [] enabledTable = Bytes.toBytes("enabledTable");
199 HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
200 htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
201
202 FileSystem filesystem = FileSystem.get(conf);
203 Path rootdir = FSUtils.getRootDir(conf);
204 FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir);
205
206 fstd.createTableDescriptor(htdEnabled);
207
208 HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(), null, null);
209 createRegion(hriEnabled, rootdir, conf, htdEnabled);
210
211 List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
212 TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
213
214 TableName disabledTable = TableName.valueOf("disabledTable");
215 HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
216 htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
217
218 fstd.createTableDescriptor(htdDisabled);
219 HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
220 createRegion(hriDisabled, rootdir, conf, htdDisabled);
221 List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
222 TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
223
224 TableName tableWithMergingRegions = TableName.valueOf("tableWithMergingRegions");
225 TEST_UTIL.createTable(tableWithMergingRegions, FAMILY, new byte [][] {Bytes.toBytes("m")});
226
227 log("Regions in hbase:meta and namespace have been created");
228
229
230
231 assertEquals(4, cluster.countServedRegions());
232
233
234 AssignmentManager am = master.getAssignmentManager();
235 RegionStates regionStates = am.getRegionStates();
236 List<HRegionInfo> mergingRegions = regionStates.getRegionsOfTable(tableWithMergingRegions);
237 assertEquals(2, mergingRegions.size());
238 HRegionInfo a = mergingRegions.get(0);
239 HRegionInfo b = mergingRegions.get(1);
240 HRegionInfo newRegion = RegionMergeTransaction.getMergedRegionInfo(a, b);
241 ServerName mergingServer = regionStates.getRegionServerOfRegion(a);
242 ServerName serverB = regionStates.getRegionServerOfRegion(b);
243 if (!serverB.equals(mergingServer)) {
244 RegionPlan plan = new RegionPlan(b, serverB, mergingServer);
245 am.balance(plan);
246 assertTrue(am.waitForAssignment(b));
247 }
248
249
250 HRegionServer hrs = cluster.getRegionServer(0);
251 ServerName serverName = hrs.getServerName();
252 HRegionInfo closingRegion = enabledRegions.remove(0);
253
254 List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
255 enabledAndAssignedRegions.add(enabledRegions.remove(0));
256 enabledAndAssignedRegions.add(enabledRegions.remove(0));
257 enabledAndAssignedRegions.add(closingRegion);
258
259 List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
260 disabledAndAssignedRegions.add(disabledRegions.remove(0));
261 disabledAndAssignedRegions.add(disabledRegions.remove(0));
262
263
264 for (HRegionInfo hri : enabledAndAssignedRegions) {
265 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
266 new RegionPlan(hri, null, serverName));
267 master.assignRegion(hri);
268 }
269 for (HRegionInfo hri : disabledAndAssignedRegions) {
270 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
271 new RegionPlan(hri, null, serverName));
272 master.assignRegion(hri);
273 }
274
275
276 log("Waiting for assignment to finish");
277 ZKAssign.blockUntilNoRIT(zkw);
278 log("Assignment completed");
279
280
281 log("Aborting master");
282 cluster.abortMaster(0);
283 cluster.waitOnMaster(0);
284 log("Master has aborted");
285
286
287
288
289
290
291 List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
292 List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
293
294 log("Beginning to mock scenarios");
295
296
297 ZKTable zktable = new ZKTable(zkw);
298 zktable.setDisabledTable(disabledTable);
299
300
301
302
303
304
305
306
307 HRegionInfo region = enabledRegions.remove(0);
308 regionsThatShouldBeOnline.add(region);
309 ZKAssign.createNodeOffline(zkw, region, serverName);
310
311
312
313
314
315 regionsThatShouldBeOnline.add(closingRegion);
316 ZKAssign.createNodeClosing(zkw, closingRegion, serverName);
317
318
319
320
321
322
323
324 region = enabledRegions.remove(0);
325 regionsThatShouldBeOnline.add(region);
326 int version = ZKAssign.createNodeClosing(zkw, region, serverName);
327 ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
328
329
330 region = disabledRegions.remove(0);
331 regionsThatShouldBeOffline.add(region);
332 version = ZKAssign.createNodeClosing(zkw, region, serverName);
333 ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
334
335
336
337
338
339
340
341 region = enabledRegions.remove(0);
342 regionsThatShouldBeOnline.add(region);
343 ZKAssign.createNodeOffline(zkw, region, serverName);
344 ProtobufUtil.openRegion(hrs, hrs.getServerName(), region);
345 while (true) {
346 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
347 RegionTransition rt = RegionTransition.parseFrom(bytes);
348 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
349 break;
350 }
351 Thread.sleep(100);
352 }
353
354
355
356 region = disabledRegions.remove(0);
357 regionsThatShouldBeOffline.add(region);
358 ZKAssign.createNodeOffline(zkw, region, serverName);
359 ProtobufUtil.openRegion(hrs, hrs.getServerName(), region);
360 while (true) {
361 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
362 RegionTransition rt = RegionTransition.parseFrom(bytes);
363 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
364 break;
365 }
366 Thread.sleep(100);
367 }
368
369
370
371
372
373
374
375 RegionMergeTransaction.createNodeMerging(
376 zkw, newRegion, mergingServer, a, b);
377
378
379
380
381
382
383
384
385
386 log("Done mocking data up in ZK");
387
388
389 log("Starting up a new master");
390 master = cluster.startMaster().getMaster();
391 log("Waiting for master to be ready");
392 cluster.waitForActiveAndReadyMaster();
393 log("Master is ready");
394
395
396 regionStates = master.getAssignmentManager().getRegionStates();
397
398 assertTrue(regionStates.isRegionInState(a, State.MERGING));
399 assertTrue(regionStates.isRegionInState(b, State.MERGING));
400 assertTrue(regionStates.isRegionInState(newRegion, State.MERGING_NEW));
401
402
403 ZKAssign.deleteNodeFailSilent(zkw, newRegion);
404
405
406 log("Waiting for no more RIT");
407 ZKAssign.blockUntilNoRIT(zkw);
408 log("No more RIT in ZK, now doing final test verification");
409
410
411 Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
412 for (JVMClusterUtil.RegionServerThread rst :
413 cluster.getRegionServerThreads()) {
414 onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rst.getRegionServer()));
415 }
416
417
418 for (HRegionInfo hri : regionsThatShouldBeOnline) {
419 assertTrue(onlineRegions.contains(hri));
420 }
421
422
423 for (HRegionInfo hri : regionsThatShouldBeOffline) {
424 if (onlineRegions.contains(hri)) {
425 LOG.debug(hri);
426 }
427 assertFalse(onlineRegions.contains(hri));
428 }
429
430 log("Done with verification, all passed, shutting down cluster");
431
432
433 TEST_UTIL.shutdownMiniCluster();
434 }
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492 @Test (timeout=180000)
493 public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
494
495 final int NUM_MASTERS = 1;
496 final int NUM_RS = 2;
497
498
499 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
500 Configuration conf = TEST_UTIL.getConfiguration();
501 conf.setBoolean("hbase.assignment.usezk", true);
502
503 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
504 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
505 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
506 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
507 log("Cluster started");
508
509
510 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
511 "unittest", new Abortable() {
512
513 @Override
514 public void abort(String why, Throwable e) {
515 LOG.error("Fatal ZK Error: " + why, e);
516 org.junit.Assert.assertFalse("Fatal ZK error", true);
517 }
518
519 @Override
520 public boolean isAborted() {
521 return false;
522 }
523
524 });
525
526
527 List<MasterThread> masterThreads = cluster.getMasterThreads();
528 assertEquals(1, masterThreads.size());
529
530
531 assertTrue(cluster.waitForActiveAndReadyMaster());
532 HMaster master = masterThreads.get(0).getMaster();
533 assertTrue(master.isActiveMaster());
534 assertTrue(master.isInitialized());
535
536
537 master.balanceSwitch(false);
538
539
540 byte [] FAMILY = Bytes.toBytes("family");
541 byte[][] SPLIT_KEYS =
542 TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 30);
543
544 byte [] enabledTable = Bytes.toBytes("enabledTable");
545 HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
546 htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
547 FileSystem filesystem = FileSystem.get(conf);
548 Path rootdir = FSUtils.getRootDir(conf);
549 FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir);
550
551 fstd.createTableDescriptor(htdEnabled);
552 HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(),
553 null, null);
554 createRegion(hriEnabled, rootdir, conf, htdEnabled);
555
556 List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
557 TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
558
559 TableName disabledTable =
560 TableName.valueOf("disabledTable");
561 HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
562 htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
563
564 fstd.createTableDescriptor(htdDisabled);
565 HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
566 createRegion(hriDisabled, rootdir, conf, htdDisabled);
567
568 List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
569 TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
570
571 log("Regions in hbase:meta and Namespace have been created");
572
573
574 assertEquals(2, cluster.countServedRegions());
575
576
577 List<RegionServerThread> regionservers =
578 cluster.getRegionServerThreads();
579 HRegionServer hrs = regionservers.get(0).getRegionServer();
580
581
582 RegionServerThread hrsDeadThread = regionservers.get(1);
583 HRegionServer hrsDead = hrsDeadThread.getRegionServer();
584 ServerName deadServerName = hrsDead.getServerName();
585
586
587 List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
588 enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6));
589 enabledRegions.removeAll(enabledAndAssignedRegions);
590 List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
591 disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6));
592 disabledRegions.removeAll(disabledAndAssignedRegions);
593
594
595 for (HRegionInfo hri : enabledAndAssignedRegions) {
596 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
597 new RegionPlan(hri, null, hrs.getServerName()));
598 master.assignRegion(hri);
599 }
600 for (HRegionInfo hri : disabledAndAssignedRegions) {
601 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
602 new RegionPlan(hri, null, hrs.getServerName()));
603 master.assignRegion(hri);
604 }
605
606 log("Waiting for assignment to finish");
607 ZKAssign.blockUntilNoRIT(zkw);
608 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
609 log("Assignment completed");
610
611 assertTrue(" Table must be enabled.", master.getAssignmentManager()
612 .getZKTable().isEnabledTable(TableName.valueOf("enabledTable")));
613
614 List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
615 enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6));
616 enabledRegions.removeAll(enabledAndOnDeadRegions);
617 List<HRegionInfo> disabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
618 disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6));
619 disabledRegions.removeAll(disabledAndOnDeadRegions);
620
621
622 for (HRegionInfo hri : enabledAndOnDeadRegions) {
623 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
624 new RegionPlan(hri, null, deadServerName));
625 master.assignRegion(hri);
626 }
627 for (HRegionInfo hri : disabledAndOnDeadRegions) {
628 master.assignmentManager.regionPlans.put(hri.getEncodedName(),
629 new RegionPlan(hri, null, deadServerName));
630 master.assignRegion(hri);
631 }
632
633
634 log("Waiting for assignment to finish");
635 ZKAssign.blockUntilNoRIT(zkw);
636 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
637 log("Assignment completed");
638
639
640
641 verifyRegionLocation(hrs, enabledAndAssignedRegions);
642 verifyRegionLocation(hrs, disabledAndAssignedRegions);
643 verifyRegionLocation(hrsDead, enabledAndOnDeadRegions);
644 verifyRegionLocation(hrsDead, disabledAndOnDeadRegions);
645
646 assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
647 enabledAndAssignedRegions.size() >= 2);
648 assertTrue(" Didn't get enough regions of disalbedTable on live rs.",
649 disabledAndAssignedRegions.size() >= 2);
650 assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.",
651 enabledAndOnDeadRegions.size() >= 2);
652 assertTrue(" Didn't get enough regions of disalbedTable on dead rs.",
653 disabledAndOnDeadRegions.size() >= 2);
654
655
656 log("Aborting master");
657 cluster.abortMaster(0);
658 cluster.waitOnMaster(0);
659 log("Master has aborted");
660
661
662
663
664
665
666 List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
667 List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
668
669 log("Beginning to mock scenarios");
670
671
672 ZKTable zktable = new ZKTable(zkw);
673 zktable.setDisabledTable(disabledTable);
674
675 assertTrue(" The enabled table should be identified on master fail over.",
676 zktable.isEnabledTable(TableName.valueOf("enabledTable")));
677
678
679
680
681
682
683 HRegionInfo region = enabledAndOnDeadRegions.remove(0);
684 regionsThatShouldBeOnline.add(region);
685 ZKAssign.createNodeClosing(zkw, region, deadServerName);
686 LOG.debug("\n\nRegion of enabled table was CLOSING on dead RS\n" +
687 region + "\n\n");
688
689
690 region = disabledAndOnDeadRegions.remove(0);
691 regionsThatShouldBeOffline.add(region);
692 ZKAssign.createNodeClosing(zkw, region, deadServerName);
693 LOG.debug("\n\nRegion of disabled table was CLOSING on dead RS\n" +
694 region + "\n\n");
695
696
697
698
699
700
701 region = enabledAndOnDeadRegions.remove(0);
702 regionsThatShouldBeOnline.add(region);
703 int version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
704 ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
705 LOG.debug("\n\nRegion of enabled table was CLOSED on dead RS\n" +
706 region + "\n\n");
707
708
709 region = disabledAndOnDeadRegions.remove(0);
710 regionsThatShouldBeOffline.add(region);
711 version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
712 ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
713 LOG.debug("\n\nRegion of disabled table was CLOSED on dead RS\n" +
714 region + "\n\n");
715
716
717
718
719
720
721 region = enabledRegions.remove(0);
722 regionsThatShouldBeOnline.add(region);
723 ZKAssign.createNodeOffline(zkw, region, deadServerName);
724 ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
725 LOG.debug("\n\nRegion of enabled table was OPENING on dead RS\n" +
726 region + "\n\n");
727
728
729 region = disabledRegions.remove(0);
730 regionsThatShouldBeOffline.add(region);
731 ZKAssign.createNodeOffline(zkw, region, deadServerName);
732 ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
733 LOG.debug("\n\nRegion of disabled table was OPENING on dead RS\n" +
734 region + "\n\n");
735
736
737
738
739
740
741 region = enabledRegions.remove(0);
742 regionsThatShouldBeOnline.add(region);
743 ZKAssign.createNodeOffline(zkw, region, deadServerName);
744 ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
745 while (true) {
746 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
747 RegionTransition rt = RegionTransition.parseFrom(bytes);
748 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
749 break;
750 }
751 Thread.sleep(100);
752 }
753 LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" +
754 region + "\n\n");
755
756
757 region = disabledRegions.remove(0);
758 regionsThatShouldBeOffline.add(region);
759 ZKAssign.createNodeOffline(zkw, region, deadServerName);
760 ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
761 while (true) {
762 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
763 RegionTransition rt = RegionTransition.parseFrom(bytes);
764 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
765 break;
766 }
767 Thread.sleep(100);
768 }
769 LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" +
770 region + "\n\n");
771
772
773
774
775
776
777 region = enabledRegions.remove(0);
778 regionsThatShouldBeOnline.add(region);
779 ZKAssign.createNodeOffline(zkw, region, deadServerName);
780 ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
781 while (true) {
782 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
783 RegionTransition rt = RegionTransition.parseFrom(bytes);
784 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
785 ZKAssign.deleteOpenedNode(zkw, region.getEncodedName(), rt.getServerName());
786 LOG.debug("DELETED " + rt);
787 break;
788 }
789 Thread.sleep(100);
790 }
791 LOG.debug("\n\nRegion of enabled table was open at steady-state on dead RS"
792 + "\n" + region + "\n\n");
793
794
795 region = disabledRegions.remove(0);
796 regionsThatShouldBeOffline.add(region);
797 ZKAssign.createNodeOffline(zkw, region, deadServerName);
798 ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
799 while (true) {
800 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
801 RegionTransition rt = RegionTransition.parseFrom(bytes);
802 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
803 ZKAssign.deleteOpenedNode(zkw, region.getEncodedName(), rt.getServerName());
804 break;
805 }
806 Thread.sleep(100);
807 }
808 LOG.debug("\n\nRegion of disabled table was open at steady-state on dead RS"
809 + "\n" + region + "\n\n");
810
811
812
813
814
815 log("Done mocking data up in ZK");
816
817
818 log("Killing RS " + deadServerName);
819 hrsDead.abort("Killing for unit test");
820 log("RS " + deadServerName + " killed");
821
822
823
824 while (hrsDeadThread.isAlive()) {
825 Threads.sleep(10);
826 }
827 log("Starting up a new master");
828 master = cluster.startMaster().getMaster();
829 log("Waiting for master to be ready");
830 assertTrue(cluster.waitForActiveAndReadyMaster());
831 log("Master is ready");
832
833
834 while (master.getServerManager().areDeadServersInProgress()) {
835 Thread.sleep(10);
836 }
837
838
839 log("Waiting for no more RIT");
840 ZKAssign.blockUntilNoRIT(zkw);
841 log("No more RIT in ZK");
842 long now = System.currentTimeMillis();
843 long maxTime = 120000;
844 boolean done = master.assignmentManager.waitUntilNoRegionsInTransition(maxTime);
845 if (!done) {
846 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
847 LOG.info("rit=" + regionStates.getRegionsInTransition());
848 }
849 long elapsed = System.currentTimeMillis() - now;
850 assertTrue("Elapsed=" + elapsed + ", maxTime=" + maxTime + ", done=" + done,
851 elapsed < maxTime);
852 log("No more RIT in RIT map, doing final test verification");
853
854
855 Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
856 now = System.currentTimeMillis();
857 maxTime = 30000;
858 for (JVMClusterUtil.RegionServerThread rst :
859 cluster.getRegionServerThreads()) {
860 try {
861 HRegionServer rs = rst.getRegionServer();
862 while (!rs.getRegionsInTransitionInRS().isEmpty()) {
863 elapsed = System.currentTimeMillis() - now;
864 assertTrue("Test timed out in getting online regions", elapsed < maxTime);
865 if (rs.isAborted() || rs.isStopped()) {
866
867 break;
868 }
869 Thread.sleep(100);
870 }
871 onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rs));
872 } catch (RegionServerStoppedException e) {
873 LOG.info("Got RegionServerStoppedException", e);
874 }
875 }
876
877
878 for (HRegionInfo hri : regionsThatShouldBeOnline) {
879 assertTrue("region=" + hri.getRegionNameAsString() + ", " + onlineRegions.toString(),
880 onlineRegions.contains(hri));
881 }
882
883
884 for (HRegionInfo hri : regionsThatShouldBeOffline) {
885 assertFalse(onlineRegions.contains(hri));
886 }
887
888 log("Done with verification, all passed, shutting down cluster");
889
890
891 TEST_UTIL.shutdownMiniCluster();
892 }
893
894
895
896
897 private void verifyRegionLocation(HRegionServer hrs, List<HRegionInfo> regions)
898 throws IOException {
899 List<HRegionInfo> tmpOnlineRegions = ProtobufUtil.getOnlineRegions(hrs);
900 Iterator<HRegionInfo> itr = regions.iterator();
901 while (itr.hasNext()) {
902 HRegionInfo tmp = itr.next();
903 if (!tmpOnlineRegions.contains(tmp)) {
904 itr.remove();
905 }
906 }
907 }
908
909 HRegion createRegion(final HRegionInfo hri, final Path rootdir, final Configuration c,
910 final HTableDescriptor htd)
911 throws IOException {
912 HRegion r = HRegion.createHRegion(hri, rootdir, c, htd);
913
914
915
916
917
918 HRegion.closeHRegion(r);
919 return r;
920 }
921
922
923
924
925 private void log(String string) {
926 LOG.info("\n\n" + string + " \n\n");
927 }
928
929 @Test (timeout=180000)
930 public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
931 throws Exception {
932 LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
933 final int NUM_MASTERS = 1;
934 final int NUM_RS = 2;
935
936
937 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
938 Configuration conf = TEST_UTIL.getConfiguration();
939 conf.setInt("hbase.master.info.port", -1);
940 conf.setBoolean("hbase.assignment.usezk", true);
941
942 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
943 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
944
945
946 List<RegionServerThread> regionServerThreads =
947 cluster.getRegionServerThreads();
948 int count = -1;
949 HRegion metaRegion = null;
950 for (RegionServerThread regionServerThread : regionServerThreads) {
951 HRegionServer regionServer = regionServerThread.getRegionServer();
952 metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
953 count++;
954 regionServer.abort("");
955 if (null != metaRegion) break;
956 }
957 HRegionServer regionServer = cluster.getRegionServer(count);
958
959 TEST_UTIL.shutdownMiniHBaseCluster();
960
961
962 ZooKeeperWatcher zkw =
963 HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL,
964 metaRegion, regionServer.getServerName());
965
966 LOG.info("Staring cluster for second time");
967 TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
968
969 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
970 while (!master.isInitialized()) {
971 Thread.sleep(100);
972 }
973
974 log("Waiting for no more RIT");
975 ZKAssign.blockUntilNoRIT(zkw);
976
977 zkw.close();
978
979 TEST_UTIL.shutdownMiniCluster();
980 }
981
982
983
984
985 @Test(timeout=240000)
986 public void testOfflineRegionReAssginedAfterMasterRestart() throws Exception {
987 final TableName table = TableName.valueOf("testOfflineRegionReAssginedAfterMasterRestart");
988 final int NUM_MASTERS = 1;
989 final int NUM_RS = 2;
990
991
992 Configuration conf = HBaseConfiguration.create();
993 conf.setBoolean("hbase.assignment.usezk", true);
994
995
996 final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
997 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
998 log("Cluster started");
999
1000 TEST_UTIL.createTable(table, Bytes.toBytes("family"));
1001 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
1002 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
1003 HRegionInfo hri = regionStates.getRegionsOfTable(table).get(0);
1004 ServerName serverName = regionStates.getRegionServerOfRegion(hri);
1005 TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
1006
1007 ServerName dstName = null;
1008 for (ServerName tmpServer : master.serverManager.getOnlineServers().keySet()) {
1009 if (!tmpServer.equals(serverName)) {
1010 dstName = tmpServer;
1011 break;
1012 }
1013 }
1014
1015 assertTrue(dstName != null);
1016
1017 TEST_UTIL.shutdownMiniHBaseCluster();
1018
1019 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
1020 ZKAssign.createNodeOffline(zkw, hri, dstName);
1021 Stat stat = new Stat();
1022 byte[] data =
1023 ZKAssign.getDataNoWatch(zkw, hri.getEncodedName(), stat);
1024 assertTrue(data != null);
1025 RegionTransition rt = RegionTransition.parseFrom(data);
1026 assertTrue(rt.getEventType() == EventType.M_ZK_REGION_OFFLINE);
1027
1028 LOG.info(hri.getEncodedName() + " region is in offline state with source server=" + serverName
1029 + " and dst server=" + dstName);
1030
1031
1032 TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
1033
1034 while (true) {
1035 master = TEST_UTIL.getHBaseCluster().getMaster();
1036 if (master != null && master.isInitialized()) {
1037 ServerManager serverManager = master.getServerManager();
1038 if (!serverManager.areDeadServersInProgress()) {
1039 break;
1040 }
1041 }
1042 Thread.sleep(200);
1043 }
1044
1045
1046 master = TEST_UTIL.getHBaseCluster().getMaster();
1047 master.getAssignmentManager().waitForAssignment(hri);
1048 regionStates = master.getAssignmentManager().getRegionStates();
1049 RegionState newState = regionStates.getRegionState(hri);
1050 assertTrue(newState.isOpened());
1051 }
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061 @Test (timeout=240000)
1062 public void testSimpleMasterFailover() throws Exception {
1063
1064 final int NUM_MASTERS = 3;
1065 final int NUM_RS = 3;
1066
1067
1068 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
1069
1070 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1071 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1072
1073
1074 List<MasterThread> masterThreads = cluster.getMasterThreads();
1075
1076
1077 for (MasterThread mt : masterThreads) {
1078 assertTrue(mt.isAlive());
1079 }
1080
1081
1082 int numActive = 0;
1083 int activeIndex = -1;
1084 ServerName activeName = null;
1085 HMaster active = null;
1086 for (int i = 0; i < masterThreads.size(); i++) {
1087 if (masterThreads.get(i).getMaster().isActiveMaster()) {
1088 numActive++;
1089 activeIndex = i;
1090 active = masterThreads.get(activeIndex).getMaster();
1091 activeName = active.getServerName();
1092 }
1093 }
1094 assertEquals(1, numActive);
1095 assertEquals(NUM_MASTERS, masterThreads.size());
1096 LOG.info("Active master " + activeName);
1097
1098
1099 assertNotNull(active);
1100 ClusterStatus status = active.getClusterStatus();
1101 assertTrue(status.getMaster().equals(activeName));
1102 assertEquals(2, status.getBackupMastersSize());
1103 assertEquals(2, status.getBackupMasters().size());
1104
1105
1106 int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
1107 HMaster master = cluster.getMaster(backupIndex);
1108 LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
1109 cluster.stopMaster(backupIndex, false);
1110 cluster.waitOnMaster(backupIndex);
1111
1112
1113 for (int i = 0; i < masterThreads.size(); i++) {
1114 if (masterThreads.get(i).getMaster().isActiveMaster()) {
1115 assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
1116 activeIndex = i;
1117 active = masterThreads.get(activeIndex).getMaster();
1118 }
1119 }
1120 assertEquals(1, numActive);
1121 assertEquals(2, masterThreads.size());
1122 int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
1123 LOG.info("Active master " + active.getServerName() + " managing " + rsCount + " regions servers");
1124 assertEquals(3, rsCount);
1125
1126
1127 assertNotNull(active);
1128 status = active.getClusterStatus();
1129 assertTrue(status.getMaster().equals(activeName));
1130 assertEquals(1, status.getBackupMastersSize());
1131 assertEquals(1, status.getBackupMasters().size());
1132
1133
1134 LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
1135 cluster.stopMaster(activeIndex, false);
1136 cluster.waitOnMaster(activeIndex);
1137
1138
1139 assertTrue(cluster.waitForActiveAndReadyMaster());
1140
1141 LOG.debug("\n\nVerifying backup master is now active\n");
1142
1143 assertEquals(1, masterThreads.size());
1144
1145
1146 active = masterThreads.get(0).getMaster();
1147 assertNotNull(active);
1148 status = active.getClusterStatus();
1149 ServerName mastername = status.getMaster();
1150 assertTrue(mastername.equals(active.getServerName()));
1151 assertTrue(active.isActiveMaster());
1152 assertEquals(0, status.getBackupMastersSize());
1153 assertEquals(0, status.getBackupMasters().size());
1154 int rss = status.getServersSize();
1155 LOG.info("Active master " + mastername.getServerName() + " managing " +
1156 rss + " region servers");
1157 assertEquals(3, rss);
1158
1159
1160 TEST_UTIL.shutdownMiniCluster();
1161 }
1162
1163
1164
1165
1166 @Test (timeout=180000)
1167 public void testPendingOpenOrCloseWhenMasterFailover() throws Exception {
1168 final int NUM_MASTERS = 1;
1169 final int NUM_RS = 1;
1170
1171
1172 Configuration conf = HBaseConfiguration.create();
1173 conf.setBoolean("hbase.assignment.usezk", false);
1174
1175
1176 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
1177 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1178 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1179 log("Cluster started");
1180
1181
1182 List<MasterThread> masterThreads = cluster.getMasterThreads();
1183 assertEquals(1, masterThreads.size());
1184
1185
1186 assertTrue(cluster.waitForActiveAndReadyMaster());
1187 HMaster master = masterThreads.get(0).getMaster();
1188 assertTrue(master.isActiveMaster());
1189 assertTrue(master.isInitialized());
1190
1191
1192 HTable onlineTable = TEST_UTIL.createTable("onlineTable", "family");
1193
1194
1195 HTableDescriptor offlineTable = new HTableDescriptor(
1196 TableName.valueOf(Bytes.toBytes("offlineTable")));
1197 offlineTable.addFamily(new HColumnDescriptor(Bytes.toBytes("family")));
1198
1199 FileSystem filesystem = FileSystem.get(conf);
1200 Path rootdir = FSUtils.getRootDir(conf);
1201 FSTableDescriptors fstd = new FSTableDescriptors(filesystem, rootdir);
1202 fstd.createTableDescriptor(offlineTable);
1203
1204 HRegionInfo hriOffline = new HRegionInfo(offlineTable.getTableName(), null, null);
1205 createRegion(hriOffline, rootdir, conf, offlineTable);
1206 MetaEditor.addRegionToMeta(master.getCatalogTracker(), hriOffline);
1207
1208 log("Regions in hbase:meta and namespace have been created");
1209
1210
1211
1212 assertEquals(3, cluster.countServedRegions());
1213 HRegionInfo hriOnline = onlineTable.getRegionLocation("").getRegionInfo();
1214
1215 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
1216 RegionStateStore stateStore = master.getAssignmentManager().getRegionStateStore();
1217
1218
1219
1220 RegionState oldState = regionStates.getRegionState(hriOnline);
1221 RegionState newState = new RegionState(hriOnline, State.PENDING_CLOSE, oldState.getServerName());
1222 stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
1223
1224
1225
1226 oldState = new RegionState(hriOffline, State.OFFLINE);
1227 newState = new RegionState(hriOffline, State.PENDING_OPEN, newState.getServerName());
1228 stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
1229
1230
1231 log("Aborting master");
1232 cluster.abortMaster(0);
1233 cluster.waitOnMaster(0);
1234 log("Master has aborted");
1235
1236
1237 log("Starting up a new master");
1238 master = cluster.startMaster().getMaster();
1239 log("Waiting for master to be ready");
1240 cluster.waitForActiveAndReadyMaster();
1241 log("Master is ready");
1242
1243
1244 master.getAssignmentManager().waitUntilNoRegionsInTransition(60000);
1245
1246
1247 regionStates = master.getAssignmentManager().getRegionStates();
1248
1249
1250 assertTrue(regionStates.isRegionOnline(hriOffline));
1251 assertTrue(regionStates.isRegionOnline(hriOnline));
1252
1253 log("Done with verification, shutting down cluster");
1254
1255
1256 TEST_UTIL.shutdownMiniCluster();
1257 }
1258 }
1259