1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.Set;
31 import java.util.TreeSet;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.Abortable;
39 import org.apache.hadoop.hbase.ClusterStatus;
40 import org.apache.hadoop.hbase.HBaseConfiguration;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.HColumnDescriptor;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HRegionInfo;
45 import org.apache.hadoop.hbase.HTableDescriptor;
46 import org.apache.hadoop.hbase.LargeTests;
47 import org.apache.hadoop.hbase.MiniHBaseCluster;
48 import org.apache.hadoop.hbase.RegionTransition;
49 import org.apache.hadoop.hbase.ServerName;
50 import org.apache.hadoop.hbase.TableName;
51 import org.apache.hadoop.hbase.catalog.MetaEditor;
52 import org.apache.hadoop.hbase.client.HTable;
53 import org.apache.hadoop.hbase.executor.EventType;
54 import org.apache.hadoop.hbase.master.RegionState.State;
55 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
56 import org.apache.hadoop.hbase.regionserver.HRegion;
57 import org.apache.hadoop.hbase.regionserver.HRegionServer;
58 import org.apache.hadoop.hbase.regionserver.RegionMergeTransaction;
59 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
60 import org.apache.hadoop.hbase.util.Bytes;
61 import org.apache.hadoop.hbase.util.FSTableDescriptors;
62 import org.apache.hadoop.hbase.util.FSUtils;
63 import org.apache.hadoop.hbase.util.JVMClusterUtil;
64 import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
65 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
66 import org.apache.hadoop.hbase.util.Threads;
67 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
68 import org.apache.hadoop.hbase.zookeeper.ZKTable;
69 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
70 import org.apache.zookeeper.data.Stat;
71 import org.junit.Test;
72 import org.junit.experimental.categories.Category;
73
74 @Category(LargeTests.class)
75 public class TestMasterFailover {
76 private static final Log LOG = LogFactory.getLog(TestMasterFailover.class);
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157 @Test (timeout=240000)
158 public void testMasterFailoverWithMockedRIT() throws Exception {
159
160 final int NUM_MASTERS = 1;
161 final int NUM_RS = 3;
162
163
164 Configuration conf = HBaseConfiguration.create();
165 conf.setBoolean("hbase.assignment.usezk", true);
166
167
168 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
169 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
170 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
171 log("Cluster started");
172
173
174 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
175
176
177 List<MasterThread> masterThreads = cluster.getMasterThreads();
178 assertEquals(1, masterThreads.size());
179
180
181 assertTrue(cluster.waitForActiveAndReadyMaster());
182 HMaster master = masterThreads.get(0).getMaster();
183 assertTrue(master.isActiveMaster());
184 assertTrue(master.isInitialized());
185
186
187 master.balanceSwitch(false);
188
189
190 byte [] FAMILY = Bytes.toBytes("family");
191 byte [][] SPLIT_KEYS = new byte [][] {
192 new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
193 Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
194 Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
195 Bytes.toBytes("iii"), Bytes.toBytes("jjj")
196 };
197
198 byte [] enabledTable = Bytes.toBytes("enabledTable");
199 HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
200 htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
201
202 FileSystem filesystem = FileSystem.get(conf);
203 Path rootdir = FSUtils.getRootDir(conf);
204 FSTableDescriptors fstd = new FSTableDescriptors(conf, filesystem, rootdir);
205
206 fstd.createTableDescriptor(htdEnabled);
207
208 HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(), null, null);
209 createRegion(hriEnabled, rootdir, conf, htdEnabled);
210
211 List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
212 TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
213
214 TableName disabledTable = TableName.valueOf("disabledTable");
215 HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
216 htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
217
218 fstd.createTableDescriptor(htdDisabled);
219 HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
220 createRegion(hriDisabled, rootdir, conf, htdDisabled);
221 List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
222 TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
223
224 TableName tableWithMergingRegions = TableName.valueOf("tableWithMergingRegions");
225 TEST_UTIL.createTable(tableWithMergingRegions, FAMILY, new byte [][] {Bytes.toBytes("m")});
226
227 log("Regions in hbase:meta and namespace have been created");
228
229
230
231 assertEquals(4, cluster.countServedRegions());
232
233
234 AssignmentManager am = master.getAssignmentManager();
235 RegionStates regionStates = am.getRegionStates();
236 List<HRegionInfo> mergingRegions = regionStates.getRegionsOfTable(tableWithMergingRegions);
237 assertEquals(2, mergingRegions.size());
238 HRegionInfo a = mergingRegions.get(0);
239 HRegionInfo b = mergingRegions.get(1);
240 HRegionInfo newRegion = RegionMergeTransaction.getMergedRegionInfo(a, b);
241 ServerName mergingServer = regionStates.getRegionServerOfRegion(a);
242 ServerName serverB = regionStates.getRegionServerOfRegion(b);
243 if (!serverB.equals(mergingServer)) {
244 RegionPlan plan = new RegionPlan(b, serverB, mergingServer);
245 am.balance(plan);
246 assertTrue(am.waitForAssignment(b));
247 }
248
249
250 HRegionServer hrs = cluster.getRegionServer(0);
251 ServerName serverName = hrs.getServerName();
252 HRegionInfo closingRegion = enabledRegions.remove(0);
253
254 List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
255 enabledAndAssignedRegions.add(enabledRegions.remove(0));
256 enabledAndAssignedRegions.add(enabledRegions.remove(0));
257 enabledAndAssignedRegions.add(closingRegion);
258
259 List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
260 disabledAndAssignedRegions.add(disabledRegions.remove(0));
261 disabledAndAssignedRegions.add(disabledRegions.remove(0));
262
263
264 for (HRegionInfo hri : enabledAndAssignedRegions) {
265 master.assignmentManager.addPlan(hri.getEncodedName(),
266 new RegionPlan(hri, null, serverName));
267 master.assignRegion(hri);
268 }
269
270 for (HRegionInfo hri : disabledAndAssignedRegions) {
271 master.assignmentManager.addPlan(hri.getEncodedName(),
272 new RegionPlan(hri, null, serverName));
273 master.assignRegion(hri);
274 }
275
276
277 log("Waiting for assignment to finish");
278 ZKAssign.blockUntilNoRIT(zkw);
279 log("Assignment completed");
280
281
282 log("Aborting master");
283 cluster.abortMaster(0);
284 cluster.waitOnMaster(0);
285 log("Master has aborted");
286
287
288
289
290
291
292 List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
293 List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
294
295 log("Beginning to mock scenarios");
296
297
298 ZKTable zktable = new ZKTable(zkw);
299 zktable.setDisabledTable(disabledTable);
300
301
302
303
304
305
306
307
308 HRegionInfo region = enabledRegions.remove(0);
309 regionsThatShouldBeOnline.add(region);
310 ZKAssign.createNodeOffline(zkw, region, serverName);
311
312
313
314
315
316 regionsThatShouldBeOnline.add(closingRegion);
317 ZKAssign.createNodeClosing(zkw, closingRegion, serverName);
318
319
320
321
322
323
324
325 region = enabledRegions.remove(0);
326 regionsThatShouldBeOnline.add(region);
327 int version = ZKAssign.createNodeClosing(zkw, region, serverName);
328 ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
329
330
331 region = disabledRegions.remove(0);
332 regionsThatShouldBeOffline.add(region);
333 version = ZKAssign.createNodeClosing(zkw, region, serverName);
334 ZKAssign.transitionNodeClosed(zkw, region, serverName, version);
335
336
337
338
339
340
341
342 region = enabledRegions.remove(0);
343 regionsThatShouldBeOnline.add(region);
344 ZKAssign.createNodeOffline(zkw, region, serverName);
345 ProtobufUtil.openRegion(hrs, hrs.getServerName(), region);
346 while (true) {
347 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
348 RegionTransition rt = RegionTransition.parseFrom(bytes);
349 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
350 break;
351 }
352 Thread.sleep(100);
353 }
354
355
356
357 region = disabledRegions.remove(0);
358 regionsThatShouldBeOffline.add(region);
359 ZKAssign.createNodeOffline(zkw, region, serverName);
360 ProtobufUtil.openRegion(hrs, hrs.getServerName(), region);
361 while (true) {
362 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
363 RegionTransition rt = RegionTransition.parseFrom(bytes);
364 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
365 break;
366 }
367 Thread.sleep(100);
368 }
369
370
371
372
373
374
375
376 RegionMergeTransaction.createNodeMerging(
377 zkw, newRegion, mergingServer, a, b);
378
379
380
381
382
383
384
385
386
387 log("Done mocking data up in ZK");
388
389
390 log("Starting up a new master");
391 master = cluster.startMaster().getMaster();
392 log("Waiting for master to be ready");
393 cluster.waitForActiveAndReadyMaster();
394 log("Master is ready");
395
396
397 regionStates = master.getAssignmentManager().getRegionStates();
398
399 assertTrue(regionStates.isRegionInState(a, State.MERGING));
400 assertTrue(regionStates.isRegionInState(b, State.MERGING));
401 assertTrue(regionStates.isRegionInState(newRegion, State.MERGING_NEW));
402
403
404 ZKAssign.deleteNodeFailSilent(zkw, newRegion);
405
406
407 log("Waiting for no more RIT");
408 ZKAssign.blockUntilNoRIT(zkw);
409 log("No more RIT in ZK, now doing final test verification");
410
411
412 Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
413 for (JVMClusterUtil.RegionServerThread rst :
414 cluster.getRegionServerThreads()) {
415 onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rst.getRegionServer()));
416 }
417
418
419 for (HRegionInfo hri : regionsThatShouldBeOnline) {
420 assertTrue(onlineRegions.contains(hri));
421 }
422
423
424 for (HRegionInfo hri : regionsThatShouldBeOffline) {
425 if (onlineRegions.contains(hri)) {
426 LOG.debug(hri);
427 }
428 assertFalse(onlineRegions.contains(hri));
429 }
430
431 log("Done with verification, all passed, shutting down cluster");
432
433
434 TEST_UTIL.shutdownMiniCluster();
435 }
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493 @Test (timeout=180000)
494 public void testMasterFailoverWithMockedRITOnDeadRS() throws Exception {
495
496 final int NUM_MASTERS = 1;
497 final int NUM_RS = 2;
498
499
500 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
501 Configuration conf = TEST_UTIL.getConfiguration();
502 conf.setBoolean("hbase.assignment.usezk", true);
503
504 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
505 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
506 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
507 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
508 log("Cluster started");
509
510
511 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
512 "unittest", new Abortable() {
513
514 @Override
515 public void abort(String why, Throwable e) {
516 LOG.error("Fatal ZK Error: " + why, e);
517 org.junit.Assert.assertFalse("Fatal ZK error", true);
518 }
519
520 @Override
521 public boolean isAborted() {
522 return false;
523 }
524
525 });
526
527
528 List<MasterThread> masterThreads = cluster.getMasterThreads();
529 assertEquals(1, masterThreads.size());
530
531
532 assertTrue(cluster.waitForActiveAndReadyMaster());
533 HMaster master = masterThreads.get(0).getMaster();
534 assertTrue(master.isActiveMaster());
535 assertTrue(master.isInitialized());
536
537
538 master.balanceSwitch(false);
539
540
541 byte [] FAMILY = Bytes.toBytes("family");
542 byte[][] SPLIT_KEYS =
543 TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 30);
544
545 byte [] enabledTable = Bytes.toBytes("enabledTable");
546 HTableDescriptor htdEnabled = new HTableDescriptor(TableName.valueOf(enabledTable));
547 htdEnabled.addFamily(new HColumnDescriptor(FAMILY));
548 FileSystem filesystem = FileSystem.get(conf);
549 Path rootdir = FSUtils.getRootDir(conf);
550 FSTableDescriptors fstd = new FSTableDescriptors(conf, filesystem, rootdir);
551
552 fstd.createTableDescriptor(htdEnabled);
553 HRegionInfo hriEnabled = new HRegionInfo(htdEnabled.getTableName(),
554 null, null);
555 createRegion(hriEnabled, rootdir, conf, htdEnabled);
556
557 List<HRegionInfo> enabledRegions = TEST_UTIL.createMultiRegionsInMeta(
558 TEST_UTIL.getConfiguration(), htdEnabled, SPLIT_KEYS);
559
560 TableName disabledTable =
561 TableName.valueOf("disabledTable");
562 HTableDescriptor htdDisabled = new HTableDescriptor(disabledTable);
563 htdDisabled.addFamily(new HColumnDescriptor(FAMILY));
564
565 fstd.createTableDescriptor(htdDisabled);
566 HRegionInfo hriDisabled = new HRegionInfo(htdDisabled.getTableName(), null, null);
567 createRegion(hriDisabled, rootdir, conf, htdDisabled);
568
569 List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
570 TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
571
572 log("Regions in hbase:meta and Namespace have been created");
573
574
575 assertEquals(2, cluster.countServedRegions());
576
577
578 List<RegionServerThread> regionservers =
579 cluster.getRegionServerThreads();
580 HRegionServer hrs = regionservers.get(0).getRegionServer();
581
582
583 RegionServerThread hrsDeadThread = regionservers.get(1);
584 HRegionServer hrsDead = hrsDeadThread.getRegionServer();
585 ServerName deadServerName = hrsDead.getServerName();
586
587
588 List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
589 enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6));
590 enabledRegions.removeAll(enabledAndAssignedRegions);
591 List<HRegionInfo> disabledAndAssignedRegions = new ArrayList<HRegionInfo>();
592 disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6));
593 disabledRegions.removeAll(disabledAndAssignedRegions);
594
595
596 for (HRegionInfo hri : enabledAndAssignedRegions) {
597 master.assignmentManager.addPlan(hri.getEncodedName(),
598 new RegionPlan(hri, null, hrs.getServerName()));
599 master.assignRegion(hri);
600 }
601 for (HRegionInfo hri : disabledAndAssignedRegions) {
602 master.assignmentManager.addPlan(hri.getEncodedName(),
603 new RegionPlan(hri, null, hrs.getServerName()));
604 master.assignRegion(hri);
605 }
606
607 log("Waiting for assignment to finish");
608 ZKAssign.blockUntilNoRIT(zkw);
609 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
610 log("Assignment completed");
611
612 assertTrue(" Table must be enabled.", master.getAssignmentManager()
613 .getZKTable().isEnabledTable(TableName.valueOf("enabledTable")));
614
615 List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
616 enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6));
617 enabledRegions.removeAll(enabledAndOnDeadRegions);
618 List<HRegionInfo> disabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
619 disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6));
620 disabledRegions.removeAll(disabledAndOnDeadRegions);
621
622
623 for (HRegionInfo hri : enabledAndOnDeadRegions) {
624 master.assignmentManager.addPlan(hri.getEncodedName(),
625 new RegionPlan(hri, null, deadServerName));
626 master.assignRegion(hri);
627 }
628 for (HRegionInfo hri : disabledAndOnDeadRegions) {
629 master.assignmentManager.addPlan(hri.getEncodedName(),
630 new RegionPlan(hri, null, deadServerName));
631 master.assignRegion(hri);
632 }
633
634
635 log("Waiting for assignment to finish");
636 ZKAssign.blockUntilNoRIT(zkw);
637 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
638 log("Assignment completed");
639
640
641
642 verifyRegionLocation(hrs, enabledAndAssignedRegions);
643 verifyRegionLocation(hrs, disabledAndAssignedRegions);
644 verifyRegionLocation(hrsDead, enabledAndOnDeadRegions);
645 verifyRegionLocation(hrsDead, disabledAndOnDeadRegions);
646
647 assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
648 enabledAndAssignedRegions.size() >= 2);
649 assertTrue(" Didn't get enough regions of disalbedTable on live rs.",
650 disabledAndAssignedRegions.size() >= 2);
651 assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.",
652 enabledAndOnDeadRegions.size() >= 2);
653 assertTrue(" Didn't get enough regions of disalbedTable on dead rs.",
654 disabledAndOnDeadRegions.size() >= 2);
655
656
657 log("Aborting master");
658 cluster.abortMaster(0);
659 cluster.waitOnMaster(0);
660 log("Master has aborted");
661
662
663
664
665
666
667 List<HRegionInfo> regionsThatShouldBeOnline = new ArrayList<HRegionInfo>();
668 List<HRegionInfo> regionsThatShouldBeOffline = new ArrayList<HRegionInfo>();
669
670 log("Beginning to mock scenarios");
671
672
673 ZKTable zktable = new ZKTable(zkw);
674 zktable.setDisabledTable(disabledTable);
675
676 assertTrue(" The enabled table should be identified on master fail over.",
677 zktable.isEnabledTable(TableName.valueOf("enabledTable")));
678
679
680
681
682
683
684 HRegionInfo region = enabledAndOnDeadRegions.remove(0);
685 regionsThatShouldBeOnline.add(region);
686 ZKAssign.createNodeClosing(zkw, region, deadServerName);
687 LOG.debug("\n\nRegion of enabled table was CLOSING on dead RS\n" +
688 region + "\n\n");
689
690
691 region = disabledAndOnDeadRegions.remove(0);
692 regionsThatShouldBeOffline.add(region);
693 ZKAssign.createNodeClosing(zkw, region, deadServerName);
694 LOG.debug("\n\nRegion of disabled table was CLOSING on dead RS\n" +
695 region + "\n\n");
696
697
698
699
700
701
702 region = enabledAndOnDeadRegions.remove(0);
703 regionsThatShouldBeOnline.add(region);
704 int version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
705 ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
706 LOG.debug("\n\nRegion of enabled table was CLOSED on dead RS\n" +
707 region + "\n\n");
708
709
710 region = disabledAndOnDeadRegions.remove(0);
711 regionsThatShouldBeOffline.add(region);
712 version = ZKAssign.createNodeClosing(zkw, region, deadServerName);
713 ZKAssign.transitionNodeClosed(zkw, region, deadServerName, version);
714 LOG.debug("\n\nRegion of disabled table was CLOSED on dead RS\n" +
715 region + "\n\n");
716
717
718
719
720
721
722 region = enabledRegions.remove(0);
723 regionsThatShouldBeOnline.add(region);
724 ZKAssign.createNodeOffline(zkw, region, deadServerName);
725 ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
726 LOG.debug("\n\nRegion of enabled table was OPENING on dead RS\n" +
727 region + "\n\n");
728
729
730 region = disabledRegions.remove(0);
731 regionsThatShouldBeOffline.add(region);
732 ZKAssign.createNodeOffline(zkw, region, deadServerName);
733 ZKAssign.transitionNodeOpening(zkw, region, deadServerName);
734 LOG.debug("\n\nRegion of disabled table was OPENING on dead RS\n" +
735 region + "\n\n");
736
737
738
739
740
741
742 region = enabledRegions.remove(0);
743 regionsThatShouldBeOnline.add(region);
744 ZKAssign.createNodeOffline(zkw, region, deadServerName);
745 ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
746 while (true) {
747 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
748 RegionTransition rt = RegionTransition.parseFrom(bytes);
749 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
750 break;
751 }
752 Thread.sleep(100);
753 }
754 LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" +
755 region + "\n\n");
756
757
758 region = disabledRegions.remove(0);
759 regionsThatShouldBeOffline.add(region);
760 ZKAssign.createNodeOffline(zkw, region, deadServerName);
761 ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
762 while (true) {
763 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
764 RegionTransition rt = RegionTransition.parseFrom(bytes);
765 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
766 break;
767 }
768 Thread.sleep(100);
769 }
770 LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" +
771 region + "\n\n");
772
773
774
775
776
777
778 region = enabledRegions.remove(0);
779 regionsThatShouldBeOnline.add(region);
780 ZKAssign.createNodeOffline(zkw, region, deadServerName);
781 ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
782 while (true) {
783 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
784 RegionTransition rt = RegionTransition.parseFrom(bytes);
785 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
786 ZKAssign.deleteOpenedNode(zkw, region.getEncodedName(), rt.getServerName());
787 LOG.debug("DELETED " + rt);
788 break;
789 }
790 Thread.sleep(100);
791 }
792 LOG.debug("\n\nRegion of enabled table was open at steady-state on dead RS"
793 + "\n" + region + "\n\n");
794
795
796 region = disabledRegions.remove(0);
797 regionsThatShouldBeOffline.add(region);
798 ZKAssign.createNodeOffline(zkw, region, deadServerName);
799 ProtobufUtil.openRegion(hrsDead, hrsDead.getServerName(), region);
800 while (true) {
801 byte [] bytes = ZKAssign.getData(zkw, region.getEncodedName());
802 RegionTransition rt = RegionTransition.parseFrom(bytes);
803 if (rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_OPENED)) {
804 ZKAssign.deleteOpenedNode(zkw, region.getEncodedName(), rt.getServerName());
805 break;
806 }
807 Thread.sleep(100);
808 }
809 LOG.debug("\n\nRegion of disabled table was open at steady-state on dead RS"
810 + "\n" + region + "\n\n");
811
812
813
814
815
816 log("Done mocking data up in ZK");
817
818
819 log("Killing RS " + deadServerName);
820 hrsDead.abort("Killing for unit test");
821 log("RS " + deadServerName + " killed");
822
823
824
825 while (hrsDeadThread.isAlive()) {
826 Threads.sleep(10);
827 }
828 log("Starting up a new master");
829 master = cluster.startMaster().getMaster();
830 log("Waiting for master to be ready");
831 assertTrue(cluster.waitForActiveAndReadyMaster());
832 log("Master is ready");
833
834
835 while (master.getServerManager().areDeadServersInProgress()) {
836 Thread.sleep(10);
837 }
838
839
840 log("Waiting for no more RIT");
841 ZKAssign.blockUntilNoRIT(zkw);
842 log("No more RIT in ZK");
843 long now = System.currentTimeMillis();
844 long maxTime = 120000;
845 boolean done = master.assignmentManager.waitUntilNoRegionsInTransition(maxTime);
846 if (!done) {
847 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
848 LOG.info("rit=" + regionStates.getRegionsInTransition());
849 }
850 long elapsed = System.currentTimeMillis() - now;
851 assertTrue("Elapsed=" + elapsed + ", maxTime=" + maxTime + ", done=" + done,
852 elapsed < maxTime);
853 log("No more RIT in RIT map, doing final test verification");
854
855
856 Set<HRegionInfo> onlineRegions = new TreeSet<HRegionInfo>();
857 now = System.currentTimeMillis();
858 maxTime = 30000;
859 for (JVMClusterUtil.RegionServerThread rst :
860 cluster.getRegionServerThreads()) {
861 try {
862 HRegionServer rs = rst.getRegionServer();
863 while (!rs.getRegionsInTransitionInRS().isEmpty()) {
864 elapsed = System.currentTimeMillis() - now;
865 assertTrue("Test timed out in getting online regions", elapsed < maxTime);
866 if (rs.isAborted() || rs.isStopped()) {
867
868 break;
869 }
870 Thread.sleep(100);
871 }
872 onlineRegions.addAll(ProtobufUtil.getOnlineRegions(rs));
873 } catch (RegionServerStoppedException e) {
874 LOG.info("Got RegionServerStoppedException", e);
875 }
876 }
877
878
879 for (HRegionInfo hri : regionsThatShouldBeOnline) {
880 assertTrue("region=" + hri.getRegionNameAsString() + ", " + onlineRegions.toString(),
881 onlineRegions.contains(hri));
882 }
883
884
885 for (HRegionInfo hri : regionsThatShouldBeOffline) {
886 assertFalse(onlineRegions.contains(hri));
887 }
888
889 log("Done with verification, all passed, shutting down cluster");
890
891
892 TEST_UTIL.shutdownMiniCluster();
893 }
894
895
896
897
898 private void verifyRegionLocation(HRegionServer hrs, List<HRegionInfo> regions)
899 throws IOException {
900 List<HRegionInfo> tmpOnlineRegions = ProtobufUtil.getOnlineRegions(hrs);
901 Iterator<HRegionInfo> itr = regions.iterator();
902 while (itr.hasNext()) {
903 HRegionInfo tmp = itr.next();
904 if (!tmpOnlineRegions.contains(tmp)) {
905 itr.remove();
906 }
907 }
908 }
909
910 HRegion createRegion(final HRegionInfo hri, final Path rootdir, final Configuration c,
911 final HTableDescriptor htd)
912 throws IOException {
913 HRegion r = HRegion.createHRegion(hri, rootdir, c, htd);
914
915
916
917
918
919 HRegion.closeHRegion(r);
920 return r;
921 }
922
923
924
925
926 private void log(String string) {
927 LOG.info("\n\n" + string + " \n\n");
928 }
929
930 @Test (timeout=180000)
931 public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState()
932 throws Exception {
933 LOG.info("Starting testShouldCheckMasterFailOverWhenMETAIsInOpenedState");
934 final int NUM_MASTERS = 1;
935 final int NUM_RS = 2;
936
937
938 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
939 Configuration conf = TEST_UTIL.getConfiguration();
940 conf.setInt("hbase.master.info.port", -1);
941 conf.setBoolean("hbase.assignment.usezk", true);
942
943 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
944 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
945
946
947 List<RegionServerThread> regionServerThreads =
948 cluster.getRegionServerThreads();
949 int count = -1;
950 HRegion metaRegion = null;
951 for (RegionServerThread regionServerThread : regionServerThreads) {
952 HRegionServer regionServer = regionServerThread.getRegionServer();
953 metaRegion = regionServer.getOnlineRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
954 count++;
955 regionServer.abort("");
956 if (null != metaRegion) break;
957 }
958 HRegionServer regionServer = cluster.getRegionServer(count);
959
960 TEST_UTIL.shutdownMiniHBaseCluster();
961
962
963 ZooKeeperWatcher zkw =
964 HBaseTestingUtility.createAndForceNodeToOpenedState(TEST_UTIL,
965 metaRegion, regionServer.getServerName());
966
967 LOG.info("Staring cluster for second time");
968 TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
969
970 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
971 while (!master.isInitialized()) {
972 Thread.sleep(100);
973 }
974
975 log("Waiting for no more RIT");
976 ZKAssign.blockUntilNoRIT(zkw);
977
978 zkw.close();
979
980 TEST_UTIL.shutdownMiniCluster();
981 }
982
983
984
985
986 @Test(timeout=240000)
987 public void testOfflineRegionReAssginedAfterMasterRestart() throws Exception {
988 final TableName table = TableName.valueOf("testOfflineRegionReAssginedAfterMasterRestart");
989 final int NUM_MASTERS = 1;
990 final int NUM_RS = 2;
991
992
993 Configuration conf = HBaseConfiguration.create();
994 conf.setBoolean("hbase.assignment.usezk", true);
995
996
997 final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
998 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
999 log("Cluster started");
1000
1001 TEST_UTIL.createTable(table, Bytes.toBytes("family"));
1002 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
1003 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
1004 HRegionInfo hri = regionStates.getRegionsOfTable(table).get(0);
1005 ServerName serverName = regionStates.getRegionServerOfRegion(hri);
1006 TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
1007
1008 ServerName dstName = null;
1009 for (ServerName tmpServer : master.serverManager.getOnlineServers().keySet()) {
1010 if (!tmpServer.equals(serverName)) {
1011 dstName = tmpServer;
1012 break;
1013 }
1014 }
1015
1016 assertTrue(dstName != null);
1017
1018 TEST_UTIL.shutdownMiniHBaseCluster();
1019
1020 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
1021 ZKAssign.createNodeOffline(zkw, hri, dstName);
1022 Stat stat = new Stat();
1023 byte[] data =
1024 ZKAssign.getDataNoWatch(zkw, hri.getEncodedName(), stat);
1025 assertTrue(data != null);
1026 RegionTransition rt = RegionTransition.parseFrom(data);
1027 assertTrue(rt.getEventType() == EventType.M_ZK_REGION_OFFLINE);
1028
1029 LOG.info(hri.getEncodedName() + " region is in offline state with source server=" + serverName
1030 + " and dst server=" + dstName);
1031
1032
1033 TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, NUM_RS);
1034
1035 while (true) {
1036 master = TEST_UTIL.getHBaseCluster().getMaster();
1037 if (master != null && master.isInitialized()) {
1038 ServerManager serverManager = master.getServerManager();
1039 if (!serverManager.areDeadServersInProgress()) {
1040 break;
1041 }
1042 }
1043 Thread.sleep(200);
1044 }
1045
1046
1047 master = TEST_UTIL.getHBaseCluster().getMaster();
1048 master.getAssignmentManager().waitForAssignment(hri);
1049 regionStates = master.getAssignmentManager().getRegionStates();
1050 RegionState newState = regionStates.getRegionState(hri);
1051 assertTrue(newState.isOpened());
1052 }
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062 @Test (timeout=240000)
1063 public void testSimpleMasterFailover() throws Exception {
1064
1065 final int NUM_MASTERS = 3;
1066 final int NUM_RS = 3;
1067
1068
1069 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
1070
1071 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1072 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1073
1074
1075 List<MasterThread> masterThreads = cluster.getMasterThreads();
1076
1077
1078 for (MasterThread mt : masterThreads) {
1079 assertTrue(mt.isAlive());
1080 }
1081
1082
1083 int numActive = 0;
1084 int activeIndex = -1;
1085 ServerName activeName = null;
1086 HMaster active = null;
1087 for (int i = 0; i < masterThreads.size(); i++) {
1088 if (masterThreads.get(i).getMaster().isActiveMaster()) {
1089 numActive++;
1090 activeIndex = i;
1091 active = masterThreads.get(activeIndex).getMaster();
1092 activeName = active.getServerName();
1093 }
1094 }
1095 assertEquals(1, numActive);
1096 assertEquals(NUM_MASTERS, masterThreads.size());
1097 LOG.info("Active master " + activeName);
1098
1099
1100 assertNotNull(active);
1101 ClusterStatus status = active.getClusterStatus();
1102 assertTrue(status.getMaster().equals(activeName));
1103 assertEquals(2, status.getBackupMastersSize());
1104 assertEquals(2, status.getBackupMasters().size());
1105
1106
1107 int backupIndex = (activeIndex == 0 ? 1 : activeIndex - 1);
1108 HMaster master = cluster.getMaster(backupIndex);
1109 LOG.debug("\n\nStopping a backup master: " + master.getServerName() + "\n");
1110 cluster.stopMaster(backupIndex, false);
1111 cluster.waitOnMaster(backupIndex);
1112
1113
1114 for (int i = 0; i < masterThreads.size(); i++) {
1115 if (masterThreads.get(i).getMaster().isActiveMaster()) {
1116 assertTrue(activeName.equals(masterThreads.get(i).getMaster().getServerName()));
1117 activeIndex = i;
1118 active = masterThreads.get(activeIndex).getMaster();
1119 }
1120 }
1121 assertEquals(1, numActive);
1122 assertEquals(2, masterThreads.size());
1123 int rsCount = masterThreads.get(activeIndex).getMaster().getClusterStatus().getServersSize();
1124 LOG.info("Active master " + active.getServerName() + " managing " + rsCount + " regions servers");
1125 assertEquals(3, rsCount);
1126
1127
1128 assertNotNull(active);
1129 status = active.getClusterStatus();
1130 assertTrue(status.getMaster().equals(activeName));
1131 assertEquals(1, status.getBackupMastersSize());
1132 assertEquals(1, status.getBackupMasters().size());
1133
1134
1135 LOG.debug("\n\nStopping the active master " + active.getServerName() + "\n");
1136 cluster.stopMaster(activeIndex, false);
1137 cluster.waitOnMaster(activeIndex);
1138
1139
1140 assertTrue(cluster.waitForActiveAndReadyMaster());
1141
1142 LOG.debug("\n\nVerifying backup master is now active\n");
1143
1144 assertEquals(1, masterThreads.size());
1145
1146
1147 active = masterThreads.get(0).getMaster();
1148 assertNotNull(active);
1149 status = active.getClusterStatus();
1150 ServerName mastername = status.getMaster();
1151 assertTrue(mastername.equals(active.getServerName()));
1152 assertTrue(active.isActiveMaster());
1153 assertEquals(0, status.getBackupMastersSize());
1154 assertEquals(0, status.getBackupMasters().size());
1155 int rss = status.getServersSize();
1156 LOG.info("Active master " + mastername.getServerName() + " managing " +
1157 rss + " region servers");
1158 assertEquals(3, rss);
1159
1160
1161 TEST_UTIL.shutdownMiniCluster();
1162 }
1163
1164
1165
1166
1167 @Test (timeout=180000)
1168 public void testPendingOpenOrCloseWhenMasterFailover() throws Exception {
1169 final int NUM_MASTERS = 1;
1170 final int NUM_RS = 1;
1171
1172
1173 Configuration conf = HBaseConfiguration.create();
1174 conf.setBoolean("hbase.assignment.usezk", false);
1175
1176
1177 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
1178 TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
1179 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1180 log("Cluster started");
1181
1182
1183 List<MasterThread> masterThreads = cluster.getMasterThreads();
1184 assertEquals(1, masterThreads.size());
1185
1186
1187 assertTrue(cluster.waitForActiveAndReadyMaster());
1188 HMaster master = masterThreads.get(0).getMaster();
1189 assertTrue(master.isActiveMaster());
1190 assertTrue(master.isInitialized());
1191
1192
1193 HTable onlineTable = TEST_UTIL.createTable("onlineTable", "family");
1194
1195
1196 HTableDescriptor offlineTable = new HTableDescriptor(
1197 TableName.valueOf(Bytes.toBytes("offlineTable")));
1198 offlineTable.addFamily(new HColumnDescriptor(Bytes.toBytes("family")));
1199
1200 FileSystem filesystem = FileSystem.get(conf);
1201 Path rootdir = FSUtils.getRootDir(conf);
1202 FSTableDescriptors fstd = new FSTableDescriptors(conf, filesystem, rootdir);
1203 fstd.createTableDescriptor(offlineTable);
1204
1205 HRegionInfo hriOffline = new HRegionInfo(offlineTable.getTableName(), null, null);
1206 createRegion(hriOffline, rootdir, conf, offlineTable);
1207 MetaEditor.addRegionToMeta(master.getCatalogTracker(), hriOffline);
1208
1209 log("Regions in hbase:meta and namespace have been created");
1210
1211
1212
1213 assertEquals(3, cluster.countServedRegions());
1214 HRegionInfo hriOnline = onlineTable.getRegionLocation("").getRegionInfo();
1215
1216 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
1217 RegionStateStore stateStore = master.getAssignmentManager().getRegionStateStore();
1218
1219
1220
1221 RegionState oldState = regionStates.getRegionState(hriOnline);
1222 RegionState newState = new RegionState(hriOnline, State.PENDING_CLOSE, oldState.getServerName());
1223 stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
1224
1225
1226
1227 oldState = new RegionState(hriOffline, State.OFFLINE);
1228 newState = new RegionState(hriOffline, State.PENDING_OPEN, newState.getServerName());
1229 stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState);
1230
1231
1232 log("Aborting master");
1233 cluster.abortMaster(0);
1234 cluster.waitOnMaster(0);
1235 log("Master has aborted");
1236
1237
1238 log("Starting up a new master");
1239 master = cluster.startMaster().getMaster();
1240 log("Waiting for master to be ready");
1241 cluster.waitForActiveAndReadyMaster();
1242 log("Master is ready");
1243
1244
1245 master.getAssignmentManager().waitUntilNoRegionsInTransition(60000);
1246
1247
1248 regionStates = master.getAssignmentManager().getRegionStates();
1249
1250
1251 assertTrue(regionStates.isRegionOnline(hriOffline));
1252 assertTrue(regionStates.isRegionOnline(hriOnline));
1253
1254 log("Done with verification, shutting down cluster");
1255
1256
1257 TEST_UTIL.shutdownMiniCluster();
1258 }
1259 }
1260