1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.hbase.HBaseConfiguration;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HServerLoad;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.MediumTests;
40 import org.apache.hadoop.hbase.Server;
41 import org.apache.hadoop.hbase.ServerName;
42 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
43 import org.apache.hadoop.hbase.catalog.CatalogTracker;
44 import org.apache.hadoop.hbase.client.Get;
45 import org.apache.hadoop.hbase.client.HConnection;
46 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.client.Scan;
49 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
50 import org.apache.hadoop.hbase.executor.ExecutorService;
51 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
52 import org.apache.hadoop.hbase.executor.RegionTransitionData;
53 import org.apache.hadoop.hbase.ipc.HRegionInterface;
54 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
55 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
56 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
57 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
58 import org.apache.hadoop.hbase.util.Bytes;
59 import org.apache.hadoop.hbase.util.Pair;
60 import org.apache.hadoop.hbase.util.Threads;
61 import org.apache.hadoop.hbase.util.Writables;
62 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
63 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
64 import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
65 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
66 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
67 import org.apache.zookeeper.KeeperException;
68 import org.apache.zookeeper.KeeperException.NodeExistsException;
69 import org.apache.zookeeper.Watcher;
70 import org.junit.After;
71 import org.junit.AfterClass;
72 import org.junit.Before;
73 import org.junit.BeforeClass;
74 import org.junit.Test;
75 import org.junit.experimental.categories.Category;
76 import org.mockito.Mockito;
77 import org.mockito.internal.util.reflection.Whitebox;
78
79 import com.google.protobuf.ServiceException;
80
81
82
83
84
85 @Category(MediumTests.class)
86 public class TestAssignmentManager {
87 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
88 private static final ServerName SERVERNAME_A =
89 new ServerName("example.org", 1234, 5678);
90 private static final ServerName SERVERNAME_B =
91 new ServerName("example.org", 0, 5678);
92 private static final HRegionInfo REGIONINFO =
93 new HRegionInfo(Bytes.toBytes("t"),
94 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
95 private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
96 Bytes.toBytes("a"),Bytes.toBytes( "b"));
97 private static int assignmentCount;
98 private static boolean enabling = false;
99
100
101 private Server server;
102 private ServerManager serverManager;
103 private ZooKeeperWatcher watcher;
104 private LoadBalancer balancer;
105
106 @BeforeClass
107 public static void beforeClass() throws Exception {
108 HTU.startMiniZKCluster();
109 }
110
111 @AfterClass
112 public static void afterClass() throws IOException {
113 HTU.shutdownMiniZKCluster();
114 }
115
116 @Before
117 public void before() throws ZooKeeperConnectionException, IOException {
118
119
120
121
122
123
124 this.server = Mockito.mock(Server.class);
125 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
126 this.watcher =
127 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
128 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
129 Mockito.doThrow(new RuntimeException("Aborted")).
130 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
131
132
133
134 this.serverManager = Mockito.mock(ServerManager.class);
135 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
136 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
137 final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
138 onlineServers.put(SERVERNAME_B, new HServerLoad());
139 onlineServers.put(SERVERNAME_A, new HServerLoad());
140 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
141 new ArrayList<ServerName>(onlineServers.keySet()));
142 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
143 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
144 thenReturn(true);
145 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
146 thenReturn(true);
147
148 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)).
149 thenReturn(RegionOpeningState.OPENED);
150 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)).
151 thenReturn(RegionOpeningState.OPENED);
152 }
153
154 @After
155 public void after() throws KeeperException {
156 if (this.watcher != null) {
157
158 ZKAssign.deleteAllNodes(this.watcher);
159 this.watcher.close();
160 }
161 }
162
163
164
165
166
167
168
169
170 @Test(timeout = 5000)
171 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
172 throws IOException, KeeperException, InterruptedException {
173 AssignmentManagerWithExtrasForTesting am =
174 setUpMockedAssignmentManager(this.server, this.serverManager);
175 try {
176 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
177 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
178 while (!am.processRITInvoked) Thread.sleep(1);
179
180
181
182
183
184 int versionid =
185 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
186 assertNotSame(versionid, -1);
187 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
188
189
190
191 while (true) {
192 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
193 if (vid != versionid) {
194 versionid = vid;
195 break;
196 }
197 }
198 assertNotSame(-1, versionid);
199
200 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
201 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
202 EventType.RS_ZK_REGION_OPENING, versionid);
203 assertNotSame(-1, versionid);
204
205 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
206 SERVERNAME_B, versionid);
207 assertNotSame(-1, versionid);
208 am.gate.set(false);
209
210 ZKAssign.blockUntilNoRIT(watcher);
211 } finally {
212 am.getExecutorService().shutdown();
213 am.shutdown();
214 }
215 }
216
217 @Test(timeout = 5000)
218 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
219 throws IOException, KeeperException, InterruptedException {
220 AssignmentManagerWithExtrasForTesting am =
221 setUpMockedAssignmentManager(this.server, this.serverManager);
222 try {
223 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
224 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
225 while (!am.processRITInvoked) Thread.sleep(1);
226
227
228
229
230
231 int versionid =
232 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
233 assertNotSame(versionid, -1);
234 am.gate.set(false);
235 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
236
237
238
239 while (true) {
240 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
241 if (vid != versionid) {
242 versionid = vid;
243 break;
244 }
245 }
246 assertNotSame(-1, versionid);
247
248 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
249 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
250 EventType.RS_ZK_REGION_OPENING, versionid);
251 assertNotSame(-1, versionid);
252
253 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
254 SERVERNAME_B, versionid);
255 assertNotSame(-1, versionid);
256
257
258 ZKAssign.blockUntilNoRIT(watcher);
259 } finally {
260 am.getExecutorService().shutdown();
261 am.shutdown();
262 }
263 }
264
265 @Test(timeout = 5000)
266 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
267 throws IOException, KeeperException, InterruptedException {
268 AssignmentManagerWithExtrasForTesting am =
269 setUpMockedAssignmentManager(this.server, this.serverManager);
270 try {
271 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
272 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
273 while (!am.processRITInvoked) Thread.sleep(1);
274
275
276
277
278
279 int versionid =
280 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
281 assertNotSame(versionid, -1);
282 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
283
284 am.gate.set(false);
285
286
287 while (true) {
288 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
289 if (vid != versionid) {
290 versionid = vid;
291 break;
292 }
293 }
294 assertNotSame(-1, versionid);
295
296 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
297 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
298 EventType.RS_ZK_REGION_OPENING, versionid);
299 assertNotSame(-1, versionid);
300
301 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
302 SERVERNAME_B, versionid);
303 assertNotSame(-1, versionid);
304
305 ZKAssign.blockUntilNoRIT(watcher);
306 } finally {
307 am.getExecutorService().shutdown();
308 am.shutdown();
309 }
310 }
311
312 private void createRegionPlanAndBalance(final AssignmentManager am,
313 final ServerName from, final ServerName to, final HRegionInfo hri) {
314
315
316 am.regionOnline(hri, from);
317
318
319 am.balance(new RegionPlan(hri, from, to));
320 }
321
322
323
324
325
326
327
328
329
330 @Test(timeout = 10000)
331 public void testBalance()
332 throws IOException, KeeperException, InterruptedException {
333
334
335 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
336
337
338 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
339 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
340 .getConfiguration());
341
342 AssignmentManager am = new AssignmentManager(this.server,
343 this.serverManager, ct, balancer, executor);
344 try {
345
346
347 this.watcher.registerListenerFirst(am);
348
349
350 am.regionOnline(REGIONINFO, SERVERNAME_A);
351
352 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
353 am.balance(plan);
354
355
356
357
358
359
360 int versionid =
361 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
362 assertNotSame(versionid, -1);
363
364
365
366
367
368 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
369
370 while (true) {
371 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
372 if (vid != versionid) {
373 versionid = vid;
374 break;
375 }
376 }
377 assertNotSame(-1, versionid);
378
379 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
380 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
381 EventType.RS_ZK_REGION_OPENING, versionid);
382 assertNotSame(-1, versionid);
383
384 versionid =
385 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
386 assertNotSame(-1, versionid);
387
388 while(am.isRegionInTransition(REGIONINFO) != null) Threads.sleep(1);
389 } finally {
390 executor.shutdown();
391 am.shutdown();
392
393 ZKAssign.deleteAllNodes(this.watcher);
394 }
395 }
396
397
398
399
400
401
402 @Test
403 public void testShutdownHandler() throws KeeperException, IOException {
404
405
406 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
407
408
409 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
410 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
411 .getConfiguration());
412
413 AssignmentManager am =
414 new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
415 try {
416 processServerShutdownHandler(ct, am, false, null);
417 } finally {
418 executor.shutdown();
419 am.shutdown();
420
421 ZKAssign.deleteAllNodes(this.watcher);
422 }
423 }
424
425
426
427
428
429
430
431 @Test
432 public void testSSHWhenDisableTableInProgress()
433 throws KeeperException, IOException {
434 testCaseWithPartiallyDisabledState(TableState.DISABLING);
435 testCaseWithPartiallyDisabledState(TableState.DISABLED);
436 }
437
438
439
440
441
442
443
444
445 @Test
446 public void testSSHWhenSplitRegionInProgress()
447 throws KeeperException, IOException, Exception {
448
449 testCaseWithSplitRegionPartial(true);
450
451 testCaseWithSplitRegionPartial(false);
452
453 }
454
455 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException,
456 NodeExistsException, InterruptedException {
457
458
459 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
460
461
462 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
463
464 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager);
465
466 am.regionOnline(REGIONINFO, SERVERNAME_A);
467
468 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
469 State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
470 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
471
472 RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
473 REGIONINFO.getRegionName(), SERVERNAME_A);
474 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
475
476 ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
477
478 try {
479 processServerShutdownHandler(ct, am, regionSplitDone, null);
480
481
482
483 if(regionSplitDone){
484 assertTrue("Region state of region in SPLITTING should be removed from rit.",
485 am.regionsInTransition.isEmpty());
486 }
487 else{
488 while (!am.assignInvoked) {
489 Thread.sleep(1);
490 }
491 assertTrue("Assign should be invoked.", am.assignInvoked);
492 }
493 } finally {
494 REGIONINFO.setOffline(false);
495 REGIONINFO.setSplit(false);
496 executor.shutdown();
497 am.shutdown();
498
499 ZKAssign.deleteAllNodes(this.watcher);
500 }
501 }
502
503 private void testCaseWithPartiallyDisabledState(TableState state) throws KeeperException, IOException, NodeExistsException {
504
505
506 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
507
508
509 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
510 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
511
512 AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
513 executor);
514
515 am.regionOnline(REGIONINFO, SERVERNAME_A);
516
517 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
518 State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
519
520 if (state == TableState.DISABLING) {
521 am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
522 } else {
523 am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
524 }
525
526 RegionTransitionData data = new RegionTransitionData(EventType.M_ZK_REGION_CLOSING,
527 REGIONINFO.getRegionName(), SERVERNAME_A);
528 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
529
530 ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
531
532 try {
533 processServerShutdownHandler(ct, am, false, null);
534
535
536 assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
537
538
539 if (state == TableState.DISABLED) {
540 assertTrue("Region state of region in pending close should be removed from rit.",
541 am.regionsInTransition.isEmpty());
542 }
543 } finally {
544 executor.shutdown();
545 am.shutdown();
546
547 ZKAssign.deleteAllNodes(this.watcher);
548 }
549 }
550
551 private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am,
552 boolean splitRegion, ServerName sn)
553 throws IOException {
554
555
556 this.watcher.registerListenerFirst(am);
557
558
559 HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
560
561
562 Result r = null;
563 if (sn == null) {
564 if (splitRegion) {
565 r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
566 } else {
567 r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
568 }
569 } else {
570 if (sn.equals(SERVERNAME_A)) {
571 r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
572 } else if (sn.equals(SERVERNAME_B)) {
573 r = new Result(new KeyValue[0]);
574 }
575 }
576
577 Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
578 thenReturn(System.currentTimeMillis());
579
580 Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
581 thenReturn(new Result [] {r}, (Result [])null);
582
583
584 HConnection connection =
585 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
586 implementation, SERVERNAME_B, REGIONINFO);
587
588
589
590 Mockito.when(ct.getConnection()).thenReturn(connection);
591 Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
592
593
594
595 DeadServer deadServers = new DeadServer();
596 deadServers.add(SERVERNAME_A);
597
598 MasterServices services = Mockito.mock(MasterServices.class);
599 Mockito.when(services.getAssignmentManager()).thenReturn(am);
600 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
601 ServerShutdownHandler handler = null;
602 if (sn != null) {
603 handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false);
604 } else {
605 handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false);
606 }
607 handler.process();
608
609 }
610
611
612
613
614
615
616
617
618 private Result getMetaTableRowResult(final HRegionInfo hri,
619 final ServerName sn)
620 throws IOException {
621
622
623 List<KeyValue> kvs = new ArrayList<KeyValue>();
624 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
625 HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
626 Writables.getBytes(hri)));
627 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
628 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
629 Bytes.toBytes(sn.getHostAndPort())));
630 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
631 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
632 Bytes.toBytes(sn.getStartcode())));
633 return new Result(kvs);
634 }
635
636
637
638
639
640
641
642
643 private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
644 throws IOException {
645 hri.setOffline(true);
646 hri.setSplit(true);
647 return getMetaTableRowResult(hri, sn);
648 }
649
650
651
652
653
654
655
656 private ExecutorService startupMasterExecutor(final String name) {
657
658 ExecutorService executor = new ExecutorService(name);
659 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
660 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
661 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
662 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
663 return executor;
664 }
665
666 @Test
667 public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
668
669 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
670
671
672
673 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
674
675 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
676 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
677 .getConfiguration());
678
679 AssignmentManager am =
680 new AssignmentManager(this.server, this.serverManager, ct, balancer, null);
681 try {
682
683 unassign(am, SERVERNAME_A, hri);
684
685 ZKAssign.deleteClosingNode(this.watcher, hri);
686
687
688
689 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
690
691
692
693
694 unassign(am, SERVERNAME_A, hri);
695
696 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
697 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
698 assertTrue(am.isRegionInTransition(hri) == null);
699 } finally {
700 am.shutdown();
701 }
702 }
703
704
705
706
707
708
709
710 @Test(timeout = 5000)
711 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
712 throws IOException, KeeperException, InterruptedException, ServiceException {
713 final RecoverableZooKeeper recoverableZk = Mockito
714 .mock(RecoverableZooKeeper.class);
715 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
716 this.server, this.serverManager);
717 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
718 null) {
719 public RecoverableZooKeeper getRecoverableZooKeeper() {
720 return recoverableZk;
721 }
722 };
723 ((ZooKeeperWatcher) zkw).registerListener(am);
724 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
725 .getChildren("/hbase/unassigned", zkw);
726 am.setWatcher((ZooKeeperWatcher) zkw);
727 try {
728 am.processDeadServersAndRegionsInTransition();
729 fail("Expected to abort");
730 } catch (NullPointerException e) {
731 fail("Should not throw NPE");
732 } catch (RuntimeException e) {
733 assertEquals("Aborted", e.getLocalizedMessage());
734 }
735 }
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
754 final HRegionInfo region, final ServerName serverName)
755 throws KeeperException, IOException {
756 RegionTransitionData data =
757 new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
758 region.getRegionName(), serverName);
759
760 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
761 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
762 throw new IOException("Failed create of ephemeral " + node);
763 }
764
765
766 return transitionNodeSplitting(zkw, region, serverName, -1);
767 }
768
769
770
771 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
772 final HRegionInfo parent,
773 final ServerName serverName, final int version)
774 throws KeeperException, IOException {
775 return ZKAssign.transitionNode(zkw, parent, serverName,
776 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
777 }
778
779 private void unassign(final AssignmentManager am, final ServerName sn,
780 final HRegionInfo hri) {
781
782 am.regionOnline(hri, sn);
783
784 am.unassign(hri);
785 }
786
787
788
789
790
791
792
793
794
795
796 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
797 final ServerManager manager)
798 throws IOException, KeeperException {
799
800 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
801
802
803
804
805
806 HRegionInterface ri = Mockito.mock(HRegionInterface.class);
807
808 Result[] result = null;
809 if (enabling) {
810 result = new Result[2];
811 result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
812 result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
813 }
814 Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
815 Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
816 thenReturn(System.currentTimeMillis());
817 if (enabling) {
818 Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
819 (Result[]) null);
820
821 Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
822 getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
823 } else {
824
825 Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
826
827 Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
828 }
829
830 HConnection connection = HConnectionTestingUtility.
831 getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
832 REGIONINFO);
833
834 Mockito.when(ct.getConnection()).thenReturn(connection);
835
836 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
837 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
838 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
839 server, manager, ct, balancer, executor);
840 return am;
841 }
842
843
844
845
846
847 @Test
848 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
849 ServiceException, InterruptedException {
850 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
851 MockedLoadBalancer.class, LoadBalancer.class);
852 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
853 this.serverManager);
854 try {
855
856
857 AtomicBoolean gate = new AtomicBoolean(false);
858 if (balancer instanceof MockedLoadBalancer) {
859 ((MockedLoadBalancer) balancer).setGateVariable(gate);
860 }
861 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
862 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
863 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
864 EventType.RS_ZK_REGION_FAILED_OPEN, v);
865 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
866 RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
867 SERVERNAME_A);
868 am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
869
870 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
871 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
872 List<ServerName> serverList = new ArrayList<ServerName>(2);
873 serverList.add(SERVERNAME_B);
874 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
875 am.nodeDataChanged(path);
876
877 while (!gate.get()) {
878 Thread.sleep(10);
879 }
880
881
882 RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
883 while (newRegionPlan == null) {
884 Thread.sleep(10);
885 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
886 }
887
888
889 assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
890 assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
891 newRegionPlan.getDestination())));
892 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
893 } finally {
894 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
895 DefaultLoadBalancer.class, LoadBalancer.class);
896 am.shutdown();
897 }
898 }
899
900
901
902
903
904
905
906
907
908 @Test
909 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
910 throws KeeperException, IOException, Exception {
911 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
912 MockedLoadBalancer.class, LoadBalancer.class);
913 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
914 new HashMap<ServerName, HServerLoad>(0));
915 List<ServerName> destServers = new ArrayList<ServerName>(1);
916 destServers.add(SERVERNAME_A);
917 Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers);
918
919
920 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
921 this.serverManager);
922 AtomicBoolean gate = new AtomicBoolean(false);
923 if (balancer instanceof MockedLoadBalancer) {
924 ((MockedLoadBalancer) balancer).setGateVariable(gate);
925 }
926 try{
927
928 am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
929 am.joinCluster();
930
931 assertFalse(
932 "Assign should not be invoked for disabling table regions during clean cluster startup.",
933 gate.get());
934
935 assertTrue("Table should be disabled.",
936 am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));
937 } finally {
938 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
939 DefaultLoadBalancer.class, LoadBalancer.class);
940 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
941 am.shutdown();
942 }
943 }
944
945
946
947
948
949
950
951
952 @Test
953 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
954 enabling = true;
955 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
956 DefaultLoadBalancer.class, LoadBalancer.class);
957 Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
958 serverAndLoad.put(SERVERNAME_A, null);
959 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
960 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
961 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
962 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
963 Server server = new HMaster(HTU.getConfiguration());
964 Whitebox.setInternalState(server, "serverManager", this.serverManager);
965 assignmentCount = 0;
966 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
967 this.serverManager);
968 am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
969 HConstants.EMPTY_END_ROW), SERVERNAME_A);
970 am.gate.set(false);
971 try {
972
973 am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
974 ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
975
976 am.joinCluster();
977 while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
978 Thread.sleep(10);
979 }
980 assertEquals("Number of assignments should be equal.", 2, assignmentCount);
981 assertTrue("Table should be enabled.",
982 am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
983 } finally {
984 enabling = false;
985 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
986 am.shutdown();
987 ZKAssign.deleteAllNodes(this.watcher);
988 assignmentCount = 0;
989 }
990 }
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003 @Test
1004 public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException,
1005 ServiceException {
1006 testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true);
1007 testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false);
1008 }
1009
1010 private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline)
1011 throws IOException, KeeperException, ServiceException {
1012
1013 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1014
1015 AssignmentManagerWithExtrasForTesting am =
1016 setUpMockedAssignmentManager(this.server, this.serverManager);
1017
1018 if (regionInOffline) {
1019 ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111);
1020 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1021 State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME));
1022 } else {
1023 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1024 State.OPENING, System.currentTimeMillis(), SERVERNAME_B));
1025 }
1026
1027 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B));
1028 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1029
1030 try {
1031 processServerShutdownHandler(ct, am, false, SERVERNAME_A);
1032 processServerShutdownHandler(ct, am, false, SERVERNAME_B);
1033 if(regionInOffline){
1034 assertFalse("Assign should not be invoked.", am.assignInvoked);
1035 } else {
1036 assertTrue("Assign should be invoked.", am.assignInvoked);
1037 }
1038 } finally {
1039 am.regionsInTransition.remove(REGIONINFO.getEncodedName());
1040 am.regionPlans.remove(REGIONINFO.getEncodedName());
1041 }
1042 }
1043
1044
1045
1046
1047
1048 public static class MockedLoadBalancer extends DefaultLoadBalancer {
1049 private AtomicBoolean gate;
1050
1051 public void setGateVariable(AtomicBoolean gate) {
1052 this.gate = gate;
1053 }
1054
1055 @Override
1056 public ServerName randomAssignment(List<ServerName> servers) {
1057 ServerName randomServerName = super.randomAssignment(servers);
1058 this.gate.set(true);
1059 return randomServerName;
1060 }
1061
1062 @Override
1063 public Map<ServerName, List<HRegionInfo>> retainAssignment(
1064 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
1065 this.gate.set(true);
1066 return super.retainAssignment(regions, servers);
1067 }
1068
1069 }
1070
1071
1072
1073
1074 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1075
1076 private final ExecutorService es;
1077
1078 private final CatalogTracker ct;
1079 boolean processRITInvoked = false;
1080 boolean assignInvoked = false;
1081 AtomicBoolean gate = new AtomicBoolean(true);
1082
1083 public AssignmentManagerWithExtrasForTesting(final Server master,
1084 final ServerManager serverManager, final CatalogTracker catalogTracker,
1085 final LoadBalancer balancer, final ExecutorService service)
1086 throws KeeperException, IOException {
1087 super(master, serverManager, catalogTracker, balancer, service);
1088 this.es = service;
1089 this.ct = catalogTracker;
1090 }
1091
1092 @Override
1093 boolean processRegionInTransition(String encodedRegionName,
1094 HRegionInfo regionInfo,
1095 Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers)
1096 throws KeeperException, IOException {
1097 this.processRITInvoked = true;
1098 return super.processRegionInTransition(encodedRegionName, regionInfo,
1099 deadServers);
1100 }
1101 @Override
1102 void processRegionsInTransition(final RegionTransitionData data,
1103 final HRegionInfo regionInfo,
1104 final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
1105 final int expectedVersion) throws KeeperException {
1106 while (this.gate.get()) Threads.sleep(1);
1107 super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
1108 }
1109
1110 @Override
1111 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
1112 boolean hijack) {
1113 if (enabling) {
1114 assignmentCount++;
1115 this.regionOnline(region, SERVERNAME_A);
1116 } else {
1117 assignInvoked = true;
1118 super.assign(region, setOfflineInZK, forceNewPlan, hijack);
1119 }
1120 }
1121
1122 @Override
1123 public ServerName getRegionServerOfRegion(HRegionInfo hri) {
1124 return SERVERNAME_A;
1125 }
1126
1127
1128 void setWatcher(ZooKeeperWatcher watcher) {
1129 this.watcher = watcher;
1130 }
1131
1132
1133
1134
1135 ExecutorService getExecutorService() {
1136 return this.es;
1137 }
1138
1139
1140
1141
1142 CatalogTracker getCatalogTracker() {
1143 return this.ct;
1144 }
1145 }
1146
1147
1148
1149
1150
1151
1152
1153 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1154 final ZooKeeperWatcher watcher) {
1155
1156
1157 watcher.registerListenerFirst(am);
1158 Thread t = new Thread("RunAmJoinCluster") {
1159 public void run() {
1160
1161
1162
1163
1164
1165 am.regionsInTransition.clear();
1166 am.regionPlans.clear();
1167 try {
1168 am.joinCluster();
1169 } catch (IOException e) {
1170 throw new RuntimeException(e);
1171 } catch (KeeperException e) {
1172 throw new RuntimeException(e);
1173 } catch (InterruptedException e) {
1174 throw new RuntimeException(e);
1175 }
1176 };
1177 };
1178 t.start();
1179 while (!t.isAlive()) Threads.sleep(1);
1180 }
1181
1182 @org.junit.Rule
1183 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1184 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1185 }