1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.hbase.HBaseConfiguration;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HServerLoad;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.MediumTests;
40 import org.apache.hadoop.hbase.Server;
41 import org.apache.hadoop.hbase.ServerName;
42 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
43 import org.apache.hadoop.hbase.catalog.CatalogTracker;
44 import org.apache.hadoop.hbase.client.Get;
45 import org.apache.hadoop.hbase.client.HConnection;
46 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.client.Scan;
49 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
50 import org.apache.hadoop.hbase.executor.ExecutorService;
51 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
52 import org.apache.hadoop.hbase.executor.RegionTransitionData;
53 import org.apache.hadoop.hbase.ipc.HRegionInterface;
54 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
55 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
56 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
57 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
58 import org.apache.hadoop.hbase.util.Bytes;
59 import org.apache.hadoop.hbase.util.Pair;
60 import org.apache.hadoop.hbase.util.Threads;
61 import org.apache.hadoop.hbase.util.Writables;
62 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
63 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
64 import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
65 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
66 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
67 import org.apache.zookeeper.KeeperException;
68 import org.apache.zookeeper.KeeperException.NodeExistsException;
69 import org.apache.zookeeper.Watcher;
70 import org.junit.After;
71 import org.junit.AfterClass;
72 import org.junit.Before;
73 import org.junit.BeforeClass;
74 import org.junit.FixMethodOrder;
75 import org.junit.Test;
76 import org.junit.experimental.categories.Category;
77 import org.junit.runners.MethodSorters;
78 import org.mockito.Mockito;
79 import org.mockito.internal.util.reflection.Whitebox;
80
81 import com.google.protobuf.ServiceException;
82
83
84
85
86
87 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
88 @Category(MediumTests.class)
89 public class TestAssignmentManager {
90 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
91 private static final ServerName SERVERNAME_A =
92 new ServerName("example.org", 1234, 5678);
93 private static final ServerName SERVERNAME_B =
94 new ServerName("example.org", 0, 5678);
95 private static final HRegionInfo REGIONINFO =
96 new HRegionInfo(Bytes.toBytes("t"),
97 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
98 private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
99 Bytes.toBytes("a"),Bytes.toBytes( "b"));
100 private static int assignmentCount;
101 private static boolean enabling = false;
102
103
104 private Server server;
105 private ServerManager serverManager;
106 private ZooKeeperWatcher watcher;
107 private LoadBalancer balancer;
108
109 @BeforeClass
110 public static void beforeClass() throws Exception {
111 HTU.startMiniZKCluster();
112 }
113
114 @AfterClass
115 public static void afterClass() throws IOException {
116 HTU.shutdownMiniZKCluster();
117 }
118
119 @Before
120 public void before() throws ZooKeeperConnectionException, IOException {
121
122
123
124
125
126
127 this.server = Mockito.mock(Server.class);
128 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
129 this.watcher =
130 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
131 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
132 Mockito.doThrow(new RuntimeException("Aborted")).
133 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
134
135
136
137 this.serverManager = Mockito.mock(ServerManager.class);
138 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
139 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
140 final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
141 onlineServers.put(SERVERNAME_B, new HServerLoad());
142 onlineServers.put(SERVERNAME_A, new HServerLoad());
143 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
144 new ArrayList<ServerName>(onlineServers.keySet()));
145 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
146 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
147 thenReturn(true);
148 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
149 thenReturn(true);
150
151 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1)).
152 thenReturn(RegionOpeningState.OPENED);
153 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1)).
154 thenReturn(RegionOpeningState.OPENED);
155 }
156
157 @After
158 public void after() throws KeeperException {
159 if (this.watcher != null) {
160
161 ZKAssign.deleteAllNodes(this.watcher);
162 this.watcher.close();
163 }
164 }
165
166
167
168
169
170
171
172
173 @Test(timeout = 60000)
174 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
175 throws IOException, KeeperException, InterruptedException {
176 AssignmentManagerWithExtrasForTesting am =
177 setUpMockedAssignmentManager(this.server, this.serverManager);
178 try {
179 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
180 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
181 while (!am.processRITInvoked) Thread.sleep(1);
182
183
184
185
186
187 int versionid =
188 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
189 assertNotSame(versionid, -1);
190 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
191
192
193
194 while (true) {
195 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
196 if (vid != versionid) {
197 versionid = vid;
198 break;
199 }
200 }
201 assertNotSame(-1, versionid);
202
203 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
204 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
205 EventType.RS_ZK_REGION_OPENING, versionid);
206 assertNotSame(-1, versionid);
207
208 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
209 SERVERNAME_B, versionid);
210 assertNotSame(-1, versionid);
211 am.gate.set(false);
212
213 ZKAssign.blockUntilNoRIT(watcher);
214 } finally {
215 am.getExecutorService().shutdown();
216 am.shutdown();
217 }
218 }
219
220 @Test(timeout = 60000)
221 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
222 throws IOException, KeeperException, InterruptedException {
223 AssignmentManagerWithExtrasForTesting am =
224 setUpMockedAssignmentManager(this.server, this.serverManager);
225 try {
226 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
227 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
228 while (!am.processRITInvoked) Thread.sleep(1);
229
230
231
232
233
234 int versionid =
235 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
236 assertNotSame(versionid, -1);
237 am.gate.set(false);
238 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
239
240
241
242 while (true) {
243 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
244 if (vid != versionid) {
245 versionid = vid;
246 break;
247 }
248 }
249 assertNotSame(-1, versionid);
250
251 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
252 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
253 EventType.RS_ZK_REGION_OPENING, versionid);
254 assertNotSame(-1, versionid);
255
256 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
257 SERVERNAME_B, versionid);
258 assertNotSame(-1, versionid);
259
260
261 ZKAssign.blockUntilNoRIT(watcher);
262 } finally {
263 am.getExecutorService().shutdown();
264 am.shutdown();
265 }
266 }
267
268 @Test(timeout = 60000)
269 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
270 throws IOException, KeeperException, InterruptedException {
271 AssignmentManagerWithExtrasForTesting am =
272 setUpMockedAssignmentManager(this.server, this.serverManager);
273 try {
274 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
275 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
276 while (!am.processRITInvoked) Thread.sleep(1);
277
278
279
280
281
282 int versionid =
283 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
284 assertNotSame(versionid, -1);
285 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
286
287 am.gate.set(false);
288
289
290 while (true) {
291 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
292 if (vid != versionid) {
293 versionid = vid;
294 break;
295 }
296 }
297 assertNotSame(-1, versionid);
298
299 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
300 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
301 EventType.RS_ZK_REGION_OPENING, versionid);
302 assertNotSame(-1, versionid);
303
304 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
305 SERVERNAME_B, versionid);
306 assertNotSame(-1, versionid);
307
308 ZKAssign.blockUntilNoRIT(watcher);
309 } finally {
310 am.getExecutorService().shutdown();
311 am.shutdown();
312 }
313 }
314
315 private void createRegionPlanAndBalance(final AssignmentManager am,
316 final ServerName from, final ServerName to, final HRegionInfo hri) {
317
318
319 am.regionOnline(hri, from);
320
321
322 am.balance(new RegionPlan(hri, from, to));
323 }
324
325
326
327
328
329
330
331
332
333 @Test(timeout = 60000)
334 public void testBalance()
335 throws IOException, KeeperException, InterruptedException {
336
337
338 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
339
340
341 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
342 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
343 .getConfiguration());
344
345 AssignmentManager am = new AssignmentManager(this.server,
346 this.serverManager, ct, balancer, executor);
347 try {
348
349
350 this.watcher.registerListenerFirst(am);
351
352
353 am.regionOnline(REGIONINFO, SERVERNAME_A);
354
355 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
356 am.balance(plan);
357
358
359
360
361
362
363 int versionid =
364 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
365 assertNotSame(versionid, -1);
366
367
368
369
370
371 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
372
373 while (true) {
374 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
375 if (vid != versionid) {
376 versionid = vid;
377 break;
378 }
379 }
380 assertNotSame(-1, versionid);
381
382 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
383 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
384 EventType.RS_ZK_REGION_OPENING, versionid);
385 assertNotSame(-1, versionid);
386
387 versionid =
388 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
389 assertNotSame(-1, versionid);
390
391 while(am.isRegionInTransition(REGIONINFO) != null) Threads.sleep(1);
392 } finally {
393 executor.shutdown();
394 am.shutdown();
395
396 ZKAssign.deleteAllNodes(this.watcher);
397 }
398 }
399
400
401
402
403
404
405 @Test
406 public void testShutdownHandler() throws KeeperException, IOException {
407
408
409 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
410
411
412 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
413 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
414 .getConfiguration());
415
416 AssignmentManager am =
417 new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
418 try {
419 processServerShutdownHandler(ct, am, false, null);
420 } finally {
421 executor.shutdown();
422 am.shutdown();
423
424 ZKAssign.deleteAllNodes(this.watcher);
425 }
426 }
427
428
429
430
431
432
433
434 @Test
435 public void testSSHWhenDisableTableInProgress()
436 throws KeeperException, IOException {
437 testCaseWithPartiallyDisabledState(TableState.DISABLING, false);
438 testCaseWithPartiallyDisabledState(TableState.DISABLED, false);
439 }
440
441 @Test
442 public void testSSHWhenDisablingTableRegionsInOpeningState()
443 throws KeeperException, IOException {
444 testCaseWithPartiallyDisabledState(TableState.DISABLING, true);
445 testCaseWithPartiallyDisabledState(TableState.DISABLED, true);
446 }
447
448
449
450
451
452
453
454
455
456 @Test
457 public void testSSHWhenSplitRegionInProgress()
458 throws KeeperException, IOException, Exception {
459
460 testCaseWithSplitRegionPartial(true);
461
462 testCaseWithSplitRegionPartial(false);
463
464 }
465
466 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException,
467 NodeExistsException, InterruptedException {
468
469
470 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
471
472
473 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
474
475 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager);
476
477 am.regionOnline(REGIONINFO, SERVERNAME_A);
478
479 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
480 State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
481 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
482
483 RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
484 REGIONINFO.getRegionName(), SERVERNAME_A);
485 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
486
487 ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
488
489 try {
490 processServerShutdownHandler(ct, am, regionSplitDone, null);
491
492
493
494 if(regionSplitDone){
495 assertTrue("Region state of region in SPLITTING should be removed from rit.",
496 am.regionsInTransition.isEmpty());
497 }
498 else{
499 while (!am.assignInvoked) {
500 Thread.sleep(1);
501 }
502 assertTrue("Assign should be invoked.", am.assignInvoked);
503 }
504 } finally {
505 REGIONINFO.setOffline(false);
506 REGIONINFO.setSplit(false);
507 executor.shutdown();
508 am.shutdown();
509
510 ZKAssign.deleteAllNodes(this.watcher);
511 }
512 }
513
514 private void testCaseWithPartiallyDisabledState(TableState state, boolean opening)
515 throws KeeperException, IOException, NodeExistsException {
516
517
518 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
519
520
521 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
522 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
523
524 AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
525 executor);
526 if (opening) {
527 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
528 State.OPENING, System.currentTimeMillis(), SERVERNAME_A));
529 } else {
530
531 am.regionOnline(REGIONINFO, SERVERNAME_A);
532
533 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
534 State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
535 }
536 if (state == TableState.DISABLING) {
537 am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
538 } else {
539 am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
540 }
541 RegionTransitionData data = null;
542 if (opening) {
543 data =
544 new RegionTransitionData(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(),
545 SERVERNAME_A);
546
547 } else {
548 data =
549 new RegionTransitionData(EventType.M_ZK_REGION_CLOSING, REGIONINFO.getRegionName(),
550 SERVERNAME_A);
551 }
552 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
553
554 ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
555
556 try {
557 processServerShutdownHandler(ct, am, false, null);
558
559
560 assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
561 assertTrue("Region state of region in pending close should be removed from rit.",
562 am.regionsInTransition.isEmpty());
563 } finally {
564 executor.shutdown();
565 am.shutdown();
566
567 ZKAssign.deleteAllNodes(this.watcher);
568 }
569 }
570
571 private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am,
572 boolean splitRegion, ServerName sn)
573 throws IOException {
574
575
576 this.watcher.registerListenerFirst(am);
577
578
579 HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
580
581
582 Result r = null;
583 if (sn == null) {
584 if (splitRegion) {
585 r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
586 } else {
587 r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
588 }
589 } else {
590 if (sn.equals(SERVERNAME_A)) {
591 r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
592 } else if (sn.equals(SERVERNAME_B)) {
593 r = new Result(new KeyValue[0]);
594 }
595 }
596
597 Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
598 thenReturn(System.currentTimeMillis());
599
600 Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
601 thenReturn(new Result [] {r}, (Result [])null);
602
603
604 HConnection connection =
605 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
606 implementation, SERVERNAME_B, REGIONINFO);
607
608
609
610 Mockito.when(ct.getConnection()).thenReturn(connection);
611 Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
612
613
614
615 DeadServer deadServers = new DeadServer();
616 deadServers.add(SERVERNAME_A);
617
618 MasterServices services = Mockito.mock(MasterServices.class);
619 Mockito.when(services.getAssignmentManager()).thenReturn(am);
620 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
621 ServerShutdownHandler handler = null;
622 if (sn != null) {
623 handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false);
624 } else {
625 handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false);
626 }
627 handler.process();
628
629 }
630
631
632
633
634
635
636
637
638 private Result getMetaTableRowResult(final HRegionInfo hri,
639 final ServerName sn)
640 throws IOException {
641
642
643 List<KeyValue> kvs = new ArrayList<KeyValue>();
644 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
645 HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
646 Writables.getBytes(hri)));
647 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
648 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
649 Bytes.toBytes(sn.getHostAndPort())));
650 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
651 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
652 Bytes.toBytes(sn.getStartcode())));
653 return new Result(kvs);
654 }
655
656
657
658
659
660
661
662
663 private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
664 throws IOException {
665 hri.setOffline(true);
666 hri.setSplit(true);
667 return getMetaTableRowResult(hri, sn);
668 }
669
670
671
672
673
674
675
676 private ExecutorService startupMasterExecutor(final String name) {
677
678 ExecutorService executor = new ExecutorService(name);
679 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
680 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
681 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
682 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
683 return executor;
684 }
685
686 @Test
687 public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
688
689 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
690
691
692
693 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
694
695 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
696 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
697 .getConfiguration());
698
699 AssignmentManager am =
700 new AssignmentManager(this.server, this.serverManager, ct, balancer, null);
701 try {
702
703 unassign(am, SERVERNAME_A, hri);
704
705 ZKAssign.deleteClosingNode(this.watcher, hri);
706
707
708
709 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
710
711
712
713
714 unassign(am, SERVERNAME_A, hri);
715
716 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
717 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
718 assertTrue(am.isRegionInTransition(hri) == null);
719 } finally {
720 am.shutdown();
721 }
722 }
723
724
725
726
727
728
729
730 @Test(timeout = 60000)
731 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
732 throws IOException, KeeperException, InterruptedException, ServiceException {
733 final RecoverableZooKeeper recoverableZk = Mockito
734 .mock(RecoverableZooKeeper.class);
735 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
736 this.server, this.serverManager);
737 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
738 null) {
739 public RecoverableZooKeeper getRecoverableZooKeeper() {
740 return recoverableZk;
741 }
742 };
743 ((ZooKeeperWatcher) zkw).registerListener(am);
744 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
745 .getChildren("/hbase/unassigned", zkw);
746 am.setWatcher((ZooKeeperWatcher) zkw);
747 try {
748 am.processDeadServersAndRegionsInTransition();
749 fail("Expected to abort");
750 } catch (NullPointerException e) {
751 fail("Should not throw NPE");
752 } catch (RuntimeException e) {
753 assertEquals("Aborted", e.getLocalizedMessage());
754 }
755 }
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
774 final HRegionInfo region, final ServerName serverName)
775 throws KeeperException, IOException {
776 RegionTransitionData data =
777 new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
778 region.getRegionName(), serverName);
779
780 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
781 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
782 throw new IOException("Failed create of ephemeral " + node);
783 }
784
785
786 return transitionNodeSplitting(zkw, region, serverName, -1);
787 }
788
789
790
791 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
792 final HRegionInfo parent,
793 final ServerName serverName, final int version)
794 throws KeeperException, IOException {
795 return ZKAssign.transitionNode(zkw, parent, serverName,
796 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
797 }
798
799 private void unassign(final AssignmentManager am, final ServerName sn,
800 final HRegionInfo hri) {
801
802 am.regionOnline(hri, sn);
803
804 am.unassign(hri);
805 }
806
807
808
809
810
811
812
813
814
815
816 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
817 final ServerManager manager)
818 throws IOException, KeeperException {
819
820 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
821
822
823
824
825
826 HRegionInterface ri = Mockito.mock(HRegionInterface.class);
827
828 Result[] result = null;
829 if (enabling) {
830 result = new Result[2];
831 result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
832 result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
833 }
834 Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
835 Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
836 thenReturn(System.currentTimeMillis());
837 if (enabling) {
838 Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
839 (Result[]) null);
840
841 Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
842 getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
843 } else {
844
845 Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
846
847 Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
848 }
849
850 HConnection connection = HConnectionTestingUtility.
851 getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
852 REGIONINFO);
853
854 Mockito.when(ct.getConnection()).thenReturn(connection);
855
856 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
857 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
858 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
859 server, manager, ct, balancer, executor);
860 return am;
861 }
862
863
864
865
866
867 @Test
868 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
869 ServiceException, InterruptedException {
870 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
871 MockedLoadBalancer.class, LoadBalancer.class);
872 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
873 this.serverManager);
874 try {
875
876
877 AtomicBoolean gate = new AtomicBoolean(false);
878 if (balancer instanceof MockedLoadBalancer) {
879 ((MockedLoadBalancer) balancer).setGateVariable(gate);
880 }
881 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
882 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
883 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
884 EventType.RS_ZK_REGION_FAILED_OPEN, v);
885 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
886 RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
887 SERVERNAME_A);
888 am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
889
890 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
891 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
892 List<ServerName> serverList = new ArrayList<ServerName>(2);
893 serverList.add(SERVERNAME_B);
894 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
895 am.nodeDataChanged(path);
896
897 while (!gate.get()) {
898 Thread.sleep(10);
899 }
900
901
902 RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
903 while (newRegionPlan == null) {
904 Thread.sleep(10);
905 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
906 }
907
908
909 assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
910 assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
911 newRegionPlan.getDestination())));
912 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
913 } finally {
914 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
915 DefaultLoadBalancer.class, LoadBalancer.class);
916 am.shutdown();
917 }
918 }
919
920
921
922
923
924
925
926
927
928 @Test
929 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
930 throws KeeperException, IOException, Exception {
931 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
932 MockedLoadBalancer.class, LoadBalancer.class);
933 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
934 new HashMap<ServerName, HServerLoad>(0));
935 List<ServerName> destServers = new ArrayList<ServerName>(1);
936 destServers.add(SERVERNAME_A);
937 Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers);
938
939
940 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
941 this.serverManager);
942 AtomicBoolean gate = new AtomicBoolean(false);
943 if (balancer instanceof MockedLoadBalancer) {
944 ((MockedLoadBalancer) balancer).setGateVariable(gate);
945 }
946 try{
947
948 am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
949 am.joinCluster();
950
951 assertFalse(
952 "Assign should not be invoked for disabling table regions during clean cluster startup.",
953 gate.get());
954
955 assertTrue("Table should be disabled.",
956 am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));
957 } finally {
958 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
959 DefaultLoadBalancer.class, LoadBalancer.class);
960 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
961 am.shutdown();
962 }
963 }
964
965
966
967
968
969
970
971
972 @Test
973 public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
974 throws KeeperException, IOException, Exception {
975 List<ServerName> destServers = new ArrayList<ServerName>(1);
976 destServers.add(SERVERNAME_A);
977 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(destServers);
978 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
979 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
980 Server server = new HMaster(HTU.getConfiguration());
981 Whitebox.setInternalState(server, "serverManager", this.serverManager);
982 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
983 this.serverManager);
984 try {
985 String tableName = "dummyTable";
986 am.enablingTables.put(tableName, null);
987
988 am.getZKTable().setEnablingTable(tableName);
989 am.joinCluster();
990 assertFalse("Table should not be present in zookeeper.",
991 am.getZKTable().isTablePresent(tableName));
992 } finally {
993 }
994 }
995
996
997
998
999
1000
1001
1002
1003 @Test
1004 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
1005 enabling = true;
1006 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
1007 DefaultLoadBalancer.class, LoadBalancer.class);
1008 Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
1009 serverAndLoad.put(SERVERNAME_A, null);
1010 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
1011 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
1012 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1013 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1014 Server server = new HMaster(HTU.getConfiguration());
1015 Whitebox.setInternalState(server, "serverManager", this.serverManager);
1016 assignmentCount = 0;
1017 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1018 this.serverManager);
1019 am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
1020 HConstants.EMPTY_END_ROW), SERVERNAME_A);
1021 am.gate.set(false);
1022 try {
1023
1024 am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
1025 ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
1026
1027 am.joinCluster();
1028 while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
1029 Thread.sleep(10);
1030 }
1031 assertEquals("Number of assignments should be equal.", 2, assignmentCount);
1032 assertTrue("Table should be enabled.",
1033 am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
1034 } finally {
1035 enabling = false;
1036 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1037 am.shutdown();
1038 ZKAssign.deleteAllNodes(this.watcher);
1039 assignmentCount = 0;
1040 }
1041 }
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054 @Test
1055 public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException,
1056 ServiceException {
1057 testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true);
1058 testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false);
1059 }
1060
1061 private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline)
1062 throws IOException, KeeperException, ServiceException {
1063
1064 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1065
1066 AssignmentManagerWithExtrasForTesting am =
1067 setUpMockedAssignmentManager(this.server, this.serverManager);
1068
1069 if (regionInOffline) {
1070 ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111);
1071 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1072 State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME));
1073 } else {
1074 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1075 State.OPENING, System.currentTimeMillis(), SERVERNAME_B));
1076 }
1077
1078 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B));
1079 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1080
1081 try {
1082 processServerShutdownHandler(ct, am, false, SERVERNAME_A);
1083 processServerShutdownHandler(ct, am, false, SERVERNAME_B);
1084 if(regionInOffline){
1085 assertFalse("Assign should not be invoked.", am.assignInvoked);
1086 } else {
1087 assertTrue("Assign should be invoked.", am.assignInvoked);
1088 }
1089 } finally {
1090 am.regionsInTransition.remove(REGIONINFO.getEncodedName());
1091 am.regionPlans.remove(REGIONINFO.getEncodedName());
1092 }
1093 }
1094
1095
1096
1097
1098
1099 public static class MockedLoadBalancer extends DefaultLoadBalancer {
1100 private AtomicBoolean gate;
1101
1102 public void setGateVariable(AtomicBoolean gate) {
1103 this.gate = gate;
1104 }
1105
1106 @Override
1107 public ServerName randomAssignment(List<ServerName> servers) {
1108 ServerName randomServerName = super.randomAssignment(servers);
1109 this.gate.set(true);
1110 return randomServerName;
1111 }
1112
1113 @Override
1114 public Map<ServerName, List<HRegionInfo>> retainAssignment(
1115 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
1116 this.gate.set(true);
1117 return super.retainAssignment(regions, servers);
1118 }
1119
1120 }
1121
1122
1123
1124
1125 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1126
1127 private final ExecutorService es;
1128
1129 private final CatalogTracker ct;
1130 boolean processRITInvoked = false;
1131 boolean assignInvoked = false;
1132 AtomicBoolean gate = new AtomicBoolean(true);
1133
1134 public AssignmentManagerWithExtrasForTesting(final Server master,
1135 final ServerManager serverManager, final CatalogTracker catalogTracker,
1136 final LoadBalancer balancer, final ExecutorService service)
1137 throws KeeperException, IOException {
1138 super(master, serverManager, catalogTracker, balancer, service);
1139 this.es = service;
1140 this.ct = catalogTracker;
1141 }
1142
1143 @Override
1144 boolean processRegionInTransition(String encodedRegionName,
1145 HRegionInfo regionInfo,
1146 Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers)
1147 throws KeeperException, IOException {
1148 this.processRITInvoked = true;
1149 return super.processRegionInTransition(encodedRegionName, regionInfo,
1150 deadServers);
1151 }
1152 @Override
1153 void processRegionsInTransition(final RegionTransitionData data,
1154 final HRegionInfo regionInfo,
1155 final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
1156 final int expectedVersion) throws KeeperException {
1157 while (this.gate.get()) Threads.sleep(1);
1158 super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
1159 }
1160
1161 @Override
1162 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
1163 boolean hijack) {
1164 if (enabling) {
1165 assignmentCount++;
1166 this.regionOnline(region, SERVERNAME_A);
1167 } else {
1168 assignInvoked = true;
1169 super.assign(region, setOfflineInZK, forceNewPlan, hijack);
1170 }
1171 }
1172
1173 @Override
1174 public ServerName getRegionServerOfRegion(HRegionInfo hri) {
1175 return SERVERNAME_A;
1176 }
1177
1178
1179 void setWatcher(ZooKeeperWatcher watcher) {
1180 this.watcher = watcher;
1181 }
1182
1183
1184
1185
1186 ExecutorService getExecutorService() {
1187 return this.es;
1188 }
1189
1190
1191
1192
1193 CatalogTracker getCatalogTracker() {
1194 return this.ct;
1195 }
1196 }
1197
1198
1199
1200
1201
1202
1203
1204 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1205 final ZooKeeperWatcher watcher) {
1206
1207
1208 watcher.registerListenerFirst(am);
1209 Thread t = new Thread("RunAmJoinCluster") {
1210 public void run() {
1211
1212
1213
1214
1215
1216 am.regionsInTransition.clear();
1217 am.regionPlans.clear();
1218 try {
1219 am.joinCluster();
1220 } catch (IOException e) {
1221 throw new RuntimeException(e);
1222 } catch (KeeperException e) {
1223 throw new RuntimeException(e);
1224 } catch (InterruptedException e) {
1225 throw new RuntimeException(e);
1226 }
1227 };
1228 };
1229 t.start();
1230 while (!t.isAlive()) Threads.sleep(1);
1231 }
1232
1233 @org.junit.Rule
1234 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1235 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1236 }