1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.hbase.HBaseConfiguration;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HServerLoad;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.MediumTests;
40 import org.apache.hadoop.hbase.Server;
41 import org.apache.hadoop.hbase.ServerName;
42 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
43 import org.apache.hadoop.hbase.catalog.CatalogTracker;
44 import org.apache.hadoop.hbase.client.Get;
45 import org.apache.hadoop.hbase.client.HConnection;
46 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.client.Scan;
49 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
50 import org.apache.hadoop.hbase.executor.ExecutorService;
51 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
52 import org.apache.hadoop.hbase.executor.RegionTransitionData;
53 import org.apache.hadoop.hbase.ipc.HRegionInterface;
54 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
55 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
56 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
57 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
58 import org.apache.hadoop.hbase.util.Bytes;
59 import org.apache.hadoop.hbase.util.Pair;
60 import org.apache.hadoop.hbase.util.Threads;
61 import org.apache.hadoop.hbase.util.Writables;
62 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
63 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
64 import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
65 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
66 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
67 import org.apache.zookeeper.KeeperException;
68 import org.apache.zookeeper.KeeperException.NodeExistsException;
69 import org.apache.zookeeper.Watcher;
70 import org.junit.After;
71 import org.junit.AfterClass;
72 import org.junit.Before;
73 import org.junit.BeforeClass;
74 import org.junit.Test;
75 import org.junit.experimental.categories.Category;
76 import org.mockito.Mockito;
77 import org.mockito.internal.util.reflection.Whitebox;
78
79 import com.google.protobuf.ServiceException;
80
81
82
83
84
85 @Category(MediumTests.class)
86 public class TestAssignmentManager {
87 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
88 private static final ServerName SERVERNAME_A =
89 new ServerName("example.org", 1234, 5678);
90 private static final ServerName SERVERNAME_B =
91 new ServerName("example.org", 0, 5678);
92 private static final HRegionInfo REGIONINFO =
93 new HRegionInfo(Bytes.toBytes("t"),
94 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
95 private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
96 Bytes.toBytes("a"),Bytes.toBytes( "b"));
97 private static int assignmentCount;
98 private static boolean enabling = false;
99
100
101 private Server server;
102 private ServerManager serverManager;
103 private ZooKeeperWatcher watcher;
104 private LoadBalancer balancer;
105
106 @BeforeClass
107 public static void beforeClass() throws Exception {
108 HTU.startMiniZKCluster();
109 }
110
111 @AfterClass
112 public static void afterClass() throws IOException {
113 HTU.shutdownMiniZKCluster();
114 }
115
116 @Before
117 public void before() throws ZooKeeperConnectionException, IOException {
118
119
120
121
122
123
124 this.server = Mockito.mock(Server.class);
125 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
126 this.watcher =
127 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
128 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
129 Mockito.doThrow(new RuntimeException("Aborted")).
130 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
131
132
133
134 this.serverManager = Mockito.mock(ServerManager.class);
135 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
136 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
137 final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
138 onlineServers.put(SERVERNAME_B, new HServerLoad());
139 onlineServers.put(SERVERNAME_A, new HServerLoad());
140 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
141 new ArrayList<ServerName>(onlineServers.keySet()));
142 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
143 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
144 thenReturn(true);
145 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
146 thenReturn(true);
147
148 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)).
149 thenReturn(RegionOpeningState.OPENED);
150 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)).
151 thenReturn(RegionOpeningState.OPENED);
152 }
153
154 @After
155 public void after() throws KeeperException {
156 if (this.watcher != null) {
157
158 ZKAssign.deleteAllNodes(this.watcher);
159 this.watcher.close();
160 }
161 }
162
163
164
165
166
167
168
169
170 @Test(timeout = 60000)
171 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
172 throws IOException, KeeperException, InterruptedException {
173 AssignmentManagerWithExtrasForTesting am =
174 setUpMockedAssignmentManager(this.server, this.serverManager);
175 try {
176 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
177 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
178 while (!am.processRITInvoked) Thread.sleep(1);
179
180
181
182
183
184 int versionid =
185 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
186 assertNotSame(versionid, -1);
187 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
188
189
190
191 while (true) {
192 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
193 if (vid != versionid) {
194 versionid = vid;
195 break;
196 }
197 }
198 assertNotSame(-1, versionid);
199
200 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
201 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
202 EventType.RS_ZK_REGION_OPENING, versionid);
203 assertNotSame(-1, versionid);
204
205 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
206 SERVERNAME_B, versionid);
207 assertNotSame(-1, versionid);
208 am.gate.set(false);
209
210 ZKAssign.blockUntilNoRIT(watcher);
211 } finally {
212 am.getExecutorService().shutdown();
213 am.shutdown();
214 }
215 }
216
217 @Test(timeout = 60000)
218 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
219 throws IOException, KeeperException, InterruptedException {
220 AssignmentManagerWithExtrasForTesting am =
221 setUpMockedAssignmentManager(this.server, this.serverManager);
222 try {
223 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
224 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
225 while (!am.processRITInvoked) Thread.sleep(1);
226
227
228
229
230
231 int versionid =
232 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
233 assertNotSame(versionid, -1);
234 am.gate.set(false);
235 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
236
237
238
239 while (true) {
240 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
241 if (vid != versionid) {
242 versionid = vid;
243 break;
244 }
245 }
246 assertNotSame(-1, versionid);
247
248 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
249 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
250 EventType.RS_ZK_REGION_OPENING, versionid);
251 assertNotSame(-1, versionid);
252
253 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
254 SERVERNAME_B, versionid);
255 assertNotSame(-1, versionid);
256
257
258 ZKAssign.blockUntilNoRIT(watcher);
259 } finally {
260 am.getExecutorService().shutdown();
261 am.shutdown();
262 }
263 }
264
265 @Test(timeout = 60000)
266 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
267 throws IOException, KeeperException, InterruptedException {
268 AssignmentManagerWithExtrasForTesting am =
269 setUpMockedAssignmentManager(this.server, this.serverManager);
270 try {
271 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
272 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
273 while (!am.processRITInvoked) Thread.sleep(1);
274
275
276
277
278
279 int versionid =
280 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
281 assertNotSame(versionid, -1);
282 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
283
284 am.gate.set(false);
285
286
287 while (true) {
288 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
289 if (vid != versionid) {
290 versionid = vid;
291 break;
292 }
293 }
294 assertNotSame(-1, versionid);
295
296 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
297 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
298 EventType.RS_ZK_REGION_OPENING, versionid);
299 assertNotSame(-1, versionid);
300
301 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
302 SERVERNAME_B, versionid);
303 assertNotSame(-1, versionid);
304
305 ZKAssign.blockUntilNoRIT(watcher);
306 } finally {
307 am.getExecutorService().shutdown();
308 am.shutdown();
309 }
310 }
311
312 private void createRegionPlanAndBalance(final AssignmentManager am,
313 final ServerName from, final ServerName to, final HRegionInfo hri) {
314
315
316 am.regionOnline(hri, from);
317
318
319 am.balance(new RegionPlan(hri, from, to));
320 }
321
322
323
324
325
326
327
328
329
330 @Test(timeout = 60000)
331 public void testBalance()
332 throws IOException, KeeperException, InterruptedException {
333
334
335 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
336
337
338 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
339 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
340 .getConfiguration());
341
342 AssignmentManager am = new AssignmentManager(this.server,
343 this.serverManager, ct, balancer, executor);
344 try {
345
346
347 this.watcher.registerListenerFirst(am);
348
349
350 am.regionOnline(REGIONINFO, SERVERNAME_A);
351
352 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
353 am.balance(plan);
354
355
356
357
358
359
360 int versionid =
361 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
362 assertNotSame(versionid, -1);
363
364
365
366
367
368 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
369
370 while (true) {
371 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
372 if (vid != versionid) {
373 versionid = vid;
374 break;
375 }
376 }
377 assertNotSame(-1, versionid);
378
379 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
380 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
381 EventType.RS_ZK_REGION_OPENING, versionid);
382 assertNotSame(-1, versionid);
383
384 versionid =
385 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
386 assertNotSame(-1, versionid);
387
388 while(am.isRegionInTransition(REGIONINFO) != null) Threads.sleep(1);
389 } finally {
390 executor.shutdown();
391 am.shutdown();
392
393 ZKAssign.deleteAllNodes(this.watcher);
394 }
395 }
396
397
398
399
400
401
402 @Test
403 public void testShutdownHandler() throws KeeperException, IOException {
404
405
406 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
407
408
409 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
410 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
411 .getConfiguration());
412
413 AssignmentManager am =
414 new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
415 try {
416 processServerShutdownHandler(ct, am, false, null);
417 } finally {
418 executor.shutdown();
419 am.shutdown();
420
421 ZKAssign.deleteAllNodes(this.watcher);
422 }
423 }
424
425
426
427
428
429
430
431 @Test
432 public void testSSHWhenDisableTableInProgress()
433 throws KeeperException, IOException {
434 testCaseWithPartiallyDisabledState(TableState.DISABLING, false);
435 testCaseWithPartiallyDisabledState(TableState.DISABLED, false);
436 }
437
438 @Test
439 public void testSSHWhenDisablingTableRegionsInOpeningState()
440 throws KeeperException, IOException {
441 testCaseWithPartiallyDisabledState(TableState.DISABLING, true);
442 testCaseWithPartiallyDisabledState(TableState.DISABLED, true);
443 }
444
445
446
447
448
449
450
451
452
453 @Test
454 public void testSSHWhenSplitRegionInProgress()
455 throws KeeperException, IOException, Exception {
456
457 testCaseWithSplitRegionPartial(true);
458
459 testCaseWithSplitRegionPartial(false);
460
461 }
462
463 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException,
464 NodeExistsException, InterruptedException {
465
466
467 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
468
469
470 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
471
472 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager);
473
474 am.regionOnline(REGIONINFO, SERVERNAME_A);
475
476 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
477 State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
478 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
479
480 RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
481 REGIONINFO.getRegionName(), SERVERNAME_A);
482 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
483
484 ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
485
486 try {
487 processServerShutdownHandler(ct, am, regionSplitDone, null);
488
489
490
491 if(regionSplitDone){
492 assertTrue("Region state of region in SPLITTING should be removed from rit.",
493 am.regionsInTransition.isEmpty());
494 }
495 else{
496 while (!am.assignInvoked) {
497 Thread.sleep(1);
498 }
499 assertTrue("Assign should be invoked.", am.assignInvoked);
500 }
501 } finally {
502 REGIONINFO.setOffline(false);
503 REGIONINFO.setSplit(false);
504 executor.shutdown();
505 am.shutdown();
506
507 ZKAssign.deleteAllNodes(this.watcher);
508 }
509 }
510
511 private void testCaseWithPartiallyDisabledState(TableState state, boolean opening)
512 throws KeeperException, IOException, NodeExistsException {
513
514
515 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
516
517
518 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
519 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
520
521 AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
522 executor);
523 if (opening) {
524 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
525 State.OPENING, System.currentTimeMillis(), SERVERNAME_A));
526 } else {
527
528 am.regionOnline(REGIONINFO, SERVERNAME_A);
529
530 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
531 State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
532 }
533 if (state == TableState.DISABLING) {
534 am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
535 } else {
536 am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
537 }
538 RegionTransitionData data = null;
539 if (opening) {
540 data =
541 new RegionTransitionData(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(),
542 SERVERNAME_A);
543
544 } else {
545 data =
546 new RegionTransitionData(EventType.M_ZK_REGION_CLOSING, REGIONINFO.getRegionName(),
547 SERVERNAME_A);
548 }
549 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
550
551 ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
552
553 try {
554 processServerShutdownHandler(ct, am, false, null);
555
556
557 assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
558 assertTrue("Region state of region in pending close should be removed from rit.",
559 am.regionsInTransition.isEmpty());
560 } finally {
561 executor.shutdown();
562 am.shutdown();
563
564 ZKAssign.deleteAllNodes(this.watcher);
565 }
566 }
567
568 private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am,
569 boolean splitRegion, ServerName sn)
570 throws IOException {
571
572
573 this.watcher.registerListenerFirst(am);
574
575
576 HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
577
578
579 Result r = null;
580 if (sn == null) {
581 if (splitRegion) {
582 r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
583 } else {
584 r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
585 }
586 } else {
587 if (sn.equals(SERVERNAME_A)) {
588 r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
589 } else if (sn.equals(SERVERNAME_B)) {
590 r = new Result(new KeyValue[0]);
591 }
592 }
593
594 Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
595 thenReturn(System.currentTimeMillis());
596
597 Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
598 thenReturn(new Result [] {r}, (Result [])null);
599
600
601 HConnection connection =
602 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
603 implementation, SERVERNAME_B, REGIONINFO);
604
605
606
607 Mockito.when(ct.getConnection()).thenReturn(connection);
608 Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
609
610
611
612 DeadServer deadServers = new DeadServer();
613 deadServers.add(SERVERNAME_A);
614
615 MasterServices services = Mockito.mock(MasterServices.class);
616 Mockito.when(services.getAssignmentManager()).thenReturn(am);
617 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
618 ServerShutdownHandler handler = null;
619 if (sn != null) {
620 handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false);
621 } else {
622 handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false);
623 }
624 handler.process();
625
626 }
627
628
629
630
631
632
633
634
635 private Result getMetaTableRowResult(final HRegionInfo hri,
636 final ServerName sn)
637 throws IOException {
638
639
640 List<KeyValue> kvs = new ArrayList<KeyValue>();
641 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
642 HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
643 Writables.getBytes(hri)));
644 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
645 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
646 Bytes.toBytes(sn.getHostAndPort())));
647 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
648 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
649 Bytes.toBytes(sn.getStartcode())));
650 return new Result(kvs);
651 }
652
653
654
655
656
657
658
659
660 private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
661 throws IOException {
662 hri.setOffline(true);
663 hri.setSplit(true);
664 return getMetaTableRowResult(hri, sn);
665 }
666
667
668
669
670
671
672
673 private ExecutorService startupMasterExecutor(final String name) {
674
675 ExecutorService executor = new ExecutorService(name);
676 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
677 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
678 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
679 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
680 return executor;
681 }
682
683 @Test
684 public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
685
686 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
687
688
689
690 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
691
692 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
693 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
694 .getConfiguration());
695
696 AssignmentManager am =
697 new AssignmentManager(this.server, this.serverManager, ct, balancer, null);
698 try {
699
700 unassign(am, SERVERNAME_A, hri);
701
702 ZKAssign.deleteClosingNode(this.watcher, hri);
703
704
705
706 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
707
708
709
710
711 unassign(am, SERVERNAME_A, hri);
712
713 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
714 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
715 assertTrue(am.isRegionInTransition(hri) == null);
716 } finally {
717 am.shutdown();
718 }
719 }
720
721
722
723
724
725
726
727 @Test(timeout = 60000)
728 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
729 throws IOException, KeeperException, InterruptedException, ServiceException {
730 final RecoverableZooKeeper recoverableZk = Mockito
731 .mock(RecoverableZooKeeper.class);
732 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
733 this.server, this.serverManager);
734 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
735 null) {
736 public RecoverableZooKeeper getRecoverableZooKeeper() {
737 return recoverableZk;
738 }
739 };
740 ((ZooKeeperWatcher) zkw).registerListener(am);
741 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
742 .getChildren("/hbase/unassigned", zkw);
743 am.setWatcher((ZooKeeperWatcher) zkw);
744 try {
745 am.processDeadServersAndRegionsInTransition();
746 fail("Expected to abort");
747 } catch (NullPointerException e) {
748 fail("Should not throw NPE");
749 } catch (RuntimeException e) {
750 assertEquals("Aborted", e.getLocalizedMessage());
751 }
752 }
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
771 final HRegionInfo region, final ServerName serverName)
772 throws KeeperException, IOException {
773 RegionTransitionData data =
774 new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
775 region.getRegionName(), serverName);
776
777 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
778 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
779 throw new IOException("Failed create of ephemeral " + node);
780 }
781
782
783 return transitionNodeSplitting(zkw, region, serverName, -1);
784 }
785
786
787
788 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
789 final HRegionInfo parent,
790 final ServerName serverName, final int version)
791 throws KeeperException, IOException {
792 return ZKAssign.transitionNode(zkw, parent, serverName,
793 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
794 }
795
796 private void unassign(final AssignmentManager am, final ServerName sn,
797 final HRegionInfo hri) {
798
799 am.regionOnline(hri, sn);
800
801 am.unassign(hri);
802 }
803
804
805
806
807
808
809
810
811
812
813 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
814 final ServerManager manager)
815 throws IOException, KeeperException {
816
817 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
818
819
820
821
822
823 HRegionInterface ri = Mockito.mock(HRegionInterface.class);
824
825 Result[] result = null;
826 if (enabling) {
827 result = new Result[2];
828 result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
829 result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
830 }
831 Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
832 Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
833 thenReturn(System.currentTimeMillis());
834 if (enabling) {
835 Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
836 (Result[]) null);
837
838 Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
839 getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
840 } else {
841
842 Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
843
844 Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
845 }
846
847 HConnection connection = HConnectionTestingUtility.
848 getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
849 REGIONINFO);
850
851 Mockito.when(ct.getConnection()).thenReturn(connection);
852
853 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
854 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
855 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
856 server, manager, ct, balancer, executor);
857 return am;
858 }
859
860
861
862
863
864 @Test
865 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
866 ServiceException, InterruptedException {
867 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
868 MockedLoadBalancer.class, LoadBalancer.class);
869 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
870 this.serverManager);
871 try {
872
873
874 AtomicBoolean gate = new AtomicBoolean(false);
875 if (balancer instanceof MockedLoadBalancer) {
876 ((MockedLoadBalancer) balancer).setGateVariable(gate);
877 }
878 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
879 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
880 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
881 EventType.RS_ZK_REGION_FAILED_OPEN, v);
882 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
883 RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
884 SERVERNAME_A);
885 am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
886
887 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
888 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
889 List<ServerName> serverList = new ArrayList<ServerName>(2);
890 serverList.add(SERVERNAME_B);
891 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
892 am.nodeDataChanged(path);
893
894 while (!gate.get()) {
895 Thread.sleep(10);
896 }
897
898
899 RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
900 while (newRegionPlan == null) {
901 Thread.sleep(10);
902 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
903 }
904
905
906 assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
907 assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
908 newRegionPlan.getDestination())));
909 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
910 } finally {
911 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
912 DefaultLoadBalancer.class, LoadBalancer.class);
913 am.shutdown();
914 }
915 }
916
917
918
919
920
921
922
923
924
925 @Test
926 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
927 throws KeeperException, IOException, Exception {
928 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
929 MockedLoadBalancer.class, LoadBalancer.class);
930 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
931 new HashMap<ServerName, HServerLoad>(0));
932 List<ServerName> destServers = new ArrayList<ServerName>(1);
933 destServers.add(SERVERNAME_A);
934 Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers);
935
936
937 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
938 this.serverManager);
939 AtomicBoolean gate = new AtomicBoolean(false);
940 if (balancer instanceof MockedLoadBalancer) {
941 ((MockedLoadBalancer) balancer).setGateVariable(gate);
942 }
943 try{
944
945 am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
946 am.joinCluster();
947
948 assertFalse(
949 "Assign should not be invoked for disabling table regions during clean cluster startup.",
950 gate.get());
951
952 assertTrue("Table should be disabled.",
953 am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));
954 } finally {
955 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
956 DefaultLoadBalancer.class, LoadBalancer.class);
957 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
958 am.shutdown();
959 }
960 }
961
962
963
964
965
966
967
968
969 @Test
970 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
971 enabling = true;
972 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
973 DefaultLoadBalancer.class, LoadBalancer.class);
974 Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
975 serverAndLoad.put(SERVERNAME_A, null);
976 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
977 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
978 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
979 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
980 Server server = new HMaster(HTU.getConfiguration());
981 Whitebox.setInternalState(server, "serverManager", this.serverManager);
982 assignmentCount = 0;
983 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
984 this.serverManager);
985 am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
986 HConstants.EMPTY_END_ROW), SERVERNAME_A);
987 am.gate.set(false);
988 try {
989
990 am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
991 ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
992
993 am.joinCluster();
994 while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
995 Thread.sleep(10);
996 }
997 assertEquals("Number of assignments should be equal.", 2, assignmentCount);
998 assertTrue("Table should be enabled.",
999 am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
1000 } finally {
1001 enabling = false;
1002 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1003 am.shutdown();
1004 ZKAssign.deleteAllNodes(this.watcher);
1005 assignmentCount = 0;
1006 }
1007 }
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020 @Test
1021 public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException,
1022 ServiceException {
1023 testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true);
1024 testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false);
1025 }
1026
1027 private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline)
1028 throws IOException, KeeperException, ServiceException {
1029
1030 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1031
1032 AssignmentManagerWithExtrasForTesting am =
1033 setUpMockedAssignmentManager(this.server, this.serverManager);
1034
1035 if (regionInOffline) {
1036 ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111);
1037 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1038 State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME));
1039 } else {
1040 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1041 State.OPENING, System.currentTimeMillis(), SERVERNAME_B));
1042 }
1043
1044 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B));
1045 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1046
1047 try {
1048 processServerShutdownHandler(ct, am, false, SERVERNAME_A);
1049 processServerShutdownHandler(ct, am, false, SERVERNAME_B);
1050 if(regionInOffline){
1051 assertFalse("Assign should not be invoked.", am.assignInvoked);
1052 } else {
1053 assertTrue("Assign should be invoked.", am.assignInvoked);
1054 }
1055 } finally {
1056 am.regionsInTransition.remove(REGIONINFO.getEncodedName());
1057 am.regionPlans.remove(REGIONINFO.getEncodedName());
1058 }
1059 }
1060
1061
1062
1063
1064
1065 public static class MockedLoadBalancer extends DefaultLoadBalancer {
1066 private AtomicBoolean gate;
1067
1068 public void setGateVariable(AtomicBoolean gate) {
1069 this.gate = gate;
1070 }
1071
1072 @Override
1073 public ServerName randomAssignment(List<ServerName> servers) {
1074 ServerName randomServerName = super.randomAssignment(servers);
1075 this.gate.set(true);
1076 return randomServerName;
1077 }
1078
1079 @Override
1080 public Map<ServerName, List<HRegionInfo>> retainAssignment(
1081 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
1082 this.gate.set(true);
1083 return super.retainAssignment(regions, servers);
1084 }
1085
1086 }
1087
1088
1089
1090
1091 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1092
1093 private final ExecutorService es;
1094
1095 private final CatalogTracker ct;
1096 boolean processRITInvoked = false;
1097 boolean assignInvoked = false;
1098 AtomicBoolean gate = new AtomicBoolean(true);
1099
1100 public AssignmentManagerWithExtrasForTesting(final Server master,
1101 final ServerManager serverManager, final CatalogTracker catalogTracker,
1102 final LoadBalancer balancer, final ExecutorService service)
1103 throws KeeperException, IOException {
1104 super(master, serverManager, catalogTracker, balancer, service);
1105 this.es = service;
1106 this.ct = catalogTracker;
1107 }
1108
1109 @Override
1110 boolean processRegionInTransition(String encodedRegionName,
1111 HRegionInfo regionInfo,
1112 Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers)
1113 throws KeeperException, IOException {
1114 this.processRITInvoked = true;
1115 return super.processRegionInTransition(encodedRegionName, regionInfo,
1116 deadServers);
1117 }
1118 @Override
1119 void processRegionsInTransition(final RegionTransitionData data,
1120 final HRegionInfo regionInfo,
1121 final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
1122 final int expectedVersion) throws KeeperException {
1123 while (this.gate.get()) Threads.sleep(1);
1124 super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
1125 }
1126
1127 @Override
1128 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
1129 boolean hijack) {
1130 if (enabling) {
1131 assignmentCount++;
1132 this.regionOnline(region, SERVERNAME_A);
1133 } else {
1134 assignInvoked = true;
1135 super.assign(region, setOfflineInZK, forceNewPlan, hijack);
1136 }
1137 }
1138
1139 @Override
1140 public ServerName getRegionServerOfRegion(HRegionInfo hri) {
1141 return SERVERNAME_A;
1142 }
1143
1144
1145 void setWatcher(ZooKeeperWatcher watcher) {
1146 this.watcher = watcher;
1147 }
1148
1149
1150
1151
1152 ExecutorService getExecutorService() {
1153 return this.es;
1154 }
1155
1156
1157
1158
1159 CatalogTracker getCatalogTracker() {
1160 return this.ct;
1161 }
1162 }
1163
1164
1165
1166
1167
1168
1169
1170 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1171 final ZooKeeperWatcher watcher) {
1172
1173
1174 watcher.registerListenerFirst(am);
1175 Thread t = new Thread("RunAmJoinCluster") {
1176 public void run() {
1177
1178
1179
1180
1181
1182 am.regionsInTransition.clear();
1183 am.regionPlans.clear();
1184 try {
1185 am.joinCluster();
1186 } catch (IOException e) {
1187 throw new RuntimeException(e);
1188 } catch (KeeperException e) {
1189 throw new RuntimeException(e);
1190 } catch (InterruptedException e) {
1191 throw new RuntimeException(e);
1192 }
1193 };
1194 };
1195 t.start();
1196 while (!t.isAlive()) Threads.sleep(1);
1197 }
1198
1199 @org.junit.Rule
1200 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1201 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1202 }