1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.*;
28 import java.util.concurrent.atomic.AtomicBoolean;
29
30 import com.google.common.collect.Lists;
31 import org.apache.hadoop.hbase.HBaseConfiguration;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.HRegionInfo;
35 import org.apache.hadoop.hbase.HServerLoad;
36 import org.apache.hadoop.hbase.KeyValue;
37 import org.apache.hadoop.hbase.MediumTests;
38 import org.apache.hadoop.hbase.Server;
39 import org.apache.hadoop.hbase.ServerName;
40 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
41 import org.apache.hadoop.hbase.catalog.CatalogTracker;
42 import org.apache.hadoop.hbase.client.Get;
43 import org.apache.hadoop.hbase.client.HConnection;
44 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
45 import org.apache.hadoop.hbase.client.Result;
46 import org.apache.hadoop.hbase.client.Scan;
47 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
48 import org.apache.hadoop.hbase.executor.ExecutorService;
49 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
50 import org.apache.hadoop.hbase.executor.RegionTransitionData;
51 import org.apache.hadoop.hbase.ipc.HRegionInterface;
52 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
53 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State;
54 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
55 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.Pair;
58 import org.apache.hadoop.hbase.util.Threads;
59 import org.apache.hadoop.hbase.util.Writables;
60 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
61 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
62 import org.apache.hadoop.hbase.zookeeper.ZKTable.TableState;
63 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
64 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
65 import org.apache.zookeeper.KeeperException;
66 import org.apache.zookeeper.KeeperException.NodeExistsException;
67 import org.apache.zookeeper.Watcher;
68 import org.junit.After;
69 import org.junit.AfterClass;
70 import org.junit.Before;
71 import org.junit.BeforeClass;
72 import org.junit.FixMethodOrder;
73 import org.junit.Test;
74 import org.junit.experimental.categories.Category;
75 import org.junit.runners.MethodSorters;
76 import org.mockito.Mockito;
77 import org.mockito.internal.util.reflection.Whitebox;
78
79 import com.google.protobuf.ServiceException;
80
81
82
83
84
85 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
86 @Category(MediumTests.class)
87 public class TestAssignmentManager {
88 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
89 private static final ServerName SERVERNAME_A =
90 new ServerName("example.org", 1234, 5678);
91 private static final ServerName SERVERNAME_B =
92 new ServerName("example.org", 0, 5678);
93 private static final ServerName SERVERNAME_C =
94 new ServerName("example.org", 123, 5678);
95 private static final HRegionInfo REGIONINFO =
96 new HRegionInfo(Bytes.toBytes("t"),
97 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
98 private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
99 Bytes.toBytes("a"),Bytes.toBytes( "b"));
100 private static int assignmentCount;
101 private static boolean enabling = false;
102
103
104 private Server server;
105 private ServerManager serverManager;
106 private ZooKeeperWatcher watcher;
107 private LoadBalancer balancer;
108
109 @BeforeClass
110 public static void beforeClass() throws Exception {
111 HTU.startMiniZKCluster();
112 }
113
114 @AfterClass
115 public static void afterClass() throws IOException {
116 HTU.shutdownMiniZKCluster();
117 }
118
119 @Before
120 public void before() throws ZooKeeperConnectionException, IOException {
121
122
123
124
125
126
127 this.server = Mockito.mock(Server.class);
128 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
129 this.watcher =
130 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
131 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
132 Mockito.doThrow(new RuntimeException("Aborted")).
133 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
134
135
136
137 this.serverManager = mockManager(SERVERNAME_A, SERVERNAME_B);
138 }
139
140 private ServerManager mockManager(ServerName... servers) throws IOException {
141 ServerManager serverManager = Mockito.mock(ServerManager.class);
142 final Map<ServerName, HServerLoad> onlineServers = new HashMap<ServerName, HServerLoad>();
143 for (ServerName server : servers) {
144 Mockito.when(serverManager.isServerOnline(server)).thenReturn(true);
145 onlineServers.put(server, new HServerLoad());
146 Mockito.when(serverManager.sendRegionClose(server, REGIONINFO, -1)).thenReturn(true);
147 Mockito.when(serverManager.sendRegionOpen(server, REGIONINFO, -1)).
148 thenReturn(RegionOpeningState.OPENED);
149 }
150 Mockito.when(serverManager.getOnlineServersList()).thenReturn(
151 new ArrayList<ServerName>(onlineServers.keySet()));
152 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
153 return serverManager;
154 }
155
156 @After
157 public void after() throws KeeperException {
158 if (this.watcher != null) {
159
160 ZKAssign.deleteAllNodes(this.watcher);
161 this.watcher.close();
162 }
163 }
164
165
166
167
168
169
170
171
172 @Test(timeout = 60000)
173 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
174 throws IOException, KeeperException, InterruptedException {
175 AssignmentManagerWithExtrasForTesting am =
176 setUpMockedAssignmentManager(this.server, this.serverManager);
177 try {
178 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
179 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
180 while (!am.processRITInvoked) Thread.sleep(1);
181
182
183
184
185
186 int versionid =
187 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
188 assertNotSame(versionid, -1);
189 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
190
191
192
193 while (true) {
194 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
195 if (vid != versionid) {
196 versionid = vid;
197 break;
198 }
199 }
200 assertNotSame(-1, versionid);
201
202 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
203 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
204 EventType.RS_ZK_REGION_OPENING, versionid);
205 assertNotSame(-1, versionid);
206
207 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
208 SERVERNAME_B, versionid);
209 assertNotSame(-1, versionid);
210 am.gate.set(false);
211
212 ZKAssign.blockUntilNoRIT(watcher);
213 } finally {
214 am.getExecutorService().shutdown();
215 am.shutdown();
216 }
217 }
218
219 @Test(timeout = 60000)
220 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
221 throws IOException, KeeperException, InterruptedException {
222 AssignmentManagerWithExtrasForTesting am =
223 setUpMockedAssignmentManager(this.server, this.serverManager);
224 try {
225 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
226 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
227 while (!am.processRITInvoked) Thread.sleep(1);
228
229
230
231
232
233 int versionid =
234 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
235 assertNotSame(versionid, -1);
236 am.gate.set(false);
237 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
238
239
240
241 while (true) {
242 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
243 if (vid != versionid) {
244 versionid = vid;
245 break;
246 }
247 }
248 assertNotSame(-1, versionid);
249
250 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
251 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
252 EventType.RS_ZK_REGION_OPENING, versionid);
253 assertNotSame(-1, versionid);
254
255 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
256 SERVERNAME_B, versionid);
257 assertNotSame(-1, versionid);
258
259
260 ZKAssign.blockUntilNoRIT(watcher);
261 } finally {
262 am.getExecutorService().shutdown();
263 am.shutdown();
264 }
265 }
266
267 @Test(timeout = 60000)
268 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
269 throws IOException, KeeperException, InterruptedException {
270 AssignmentManagerWithExtrasForTesting am =
271 setUpMockedAssignmentManager(this.server, this.serverManager);
272 try {
273 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
274 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
275 while (!am.processRITInvoked) Thread.sleep(1);
276
277
278
279
280
281 int versionid =
282 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
283 assertNotSame(versionid, -1);
284 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
285
286 am.gate.set(false);
287
288
289 while (true) {
290 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
291 if (vid != versionid) {
292 versionid = vid;
293 break;
294 }
295 }
296 assertNotSame(-1, versionid);
297
298 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
299 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
300 EventType.RS_ZK_REGION_OPENING, versionid);
301 assertNotSame(-1, versionid);
302
303 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
304 SERVERNAME_B, versionid);
305 assertNotSame(-1, versionid);
306
307 ZKAssign.blockUntilNoRIT(watcher);
308 } finally {
309 am.getExecutorService().shutdown();
310 am.shutdown();
311 }
312 }
313
314 private void createRegionPlanAndBalance(final AssignmentManager am,
315 final ServerName from, final ServerName to, final HRegionInfo hri) {
316
317
318 am.regionOnline(hri, from);
319
320
321 am.balance(new RegionPlan(hri, from, to));
322 }
323
324
325
326
327
328
329
330
331
332 @Test(timeout = 60000)
333 public void testBalance()
334 throws IOException, KeeperException, InterruptedException {
335
336
337 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
338
339
340 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
341 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
342 .getConfiguration());
343
344 AssignmentManager am = new AssignmentManager(this.server,
345 this.serverManager, ct, balancer, executor);
346 try {
347
348
349 this.watcher.registerListenerFirst(am);
350
351
352 am.regionOnline(REGIONINFO, SERVERNAME_A);
353
354 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
355 am.balance(plan);
356
357
358
359
360
361
362 int versionid =
363 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
364 assertNotSame(versionid, -1);
365
366
367
368
369
370 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
371
372 while (true) {
373 int vid = ZKAssign.getVersion(this.watcher, REGIONINFO);
374 if (vid != versionid) {
375 versionid = vid;
376 break;
377 }
378 }
379 assertNotSame(-1, versionid);
380
381 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
382 SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
383 EventType.RS_ZK_REGION_OPENING, versionid);
384 assertNotSame(-1, versionid);
385
386 versionid =
387 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
388 assertNotSame(-1, versionid);
389
390 while(am.isRegionInTransition(REGIONINFO) != null) Threads.sleep(1);
391 } finally {
392 executor.shutdown();
393 am.shutdown();
394
395 ZKAssign.deleteAllNodes(this.watcher);
396 }
397 }
398
399 @Test
400 public void testGettingAssignmentsExcludesDrainingServers() throws Exception {
401 List<ServerName> availableServers =
402 Lists.newArrayList(SERVERNAME_A, SERVERNAME_B, SERVERNAME_C);
403 ServerManager serverManager = mockManager(availableServers.toArray(new ServerName[0]));
404
405
406 ExecutorService executor = startupMasterExecutor("testAssignmentsWithRSInDraining");
407 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
408
409 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
410
411 Mockito.when(serverManager.getDrainingServersList()).thenReturn(
412 Lists.newArrayList(SERVERNAME_C));
413 AssignmentManager am = new AssignmentManager(this.server, serverManager, ct, balancer, executor);
414
415 for (ServerName availableServer : availableServers) {
416 HRegionInfo info = Mockito.mock(HRegionInfo.class);
417 Mockito.when(info.getEncodedName()).thenReturn(UUID.randomUUID().toString());
418 am.regionOnline(info, availableServer);
419 }
420
421 Map<String, Map<ServerName, List<HRegionInfo>>> result = am.getAssignmentsByTable();
422 for (Map<ServerName, List<HRegionInfo>> map : result.values()) {
423 System.out.println(map.keySet());
424 assertFalse(map.containsKey(SERVERNAME_C));
425 }
426 }
427
428
429
430
431
432
433 @Test
434 public void testShutdownHandler() throws KeeperException, IOException {
435
436
437 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
438
439
440 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
441 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
442 .getConfiguration());
443
444 AssignmentManager am =
445 new AssignmentManager(this.server, this.serverManager, ct, balancer, executor);
446 try {
447 processServerShutdownHandler(ct, am, false, null);
448 } finally {
449 executor.shutdown();
450 am.shutdown();
451
452 ZKAssign.deleteAllNodes(this.watcher);
453 }
454 }
455
456
457
458
459
460
461
462 @Test
463 public void testSSHWhenDisableTableInProgress()
464 throws KeeperException, IOException {
465 testCaseWithPartiallyDisabledState(TableState.DISABLING, false);
466 testCaseWithPartiallyDisabledState(TableState.DISABLED, false);
467 }
468
469 @Test
470 public void testSSHWhenDisablingTableRegionsInOpeningState()
471 throws KeeperException, IOException {
472 testCaseWithPartiallyDisabledState(TableState.DISABLING, true);
473 testCaseWithPartiallyDisabledState(TableState.DISABLED, true);
474 }
475
476
477
478
479
480
481
482
483
484 @Test
485 public void testSSHWhenSplitRegionInProgress()
486 throws KeeperException, IOException, Exception {
487
488 testCaseWithSplitRegionPartial(true);
489
490 testCaseWithSplitRegionPartial(false);
491
492 }
493
494 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException, IOException,
495 NodeExistsException, InterruptedException {
496
497
498 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
499
500
501 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
502
503 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager);
504
505 am.regionOnline(REGIONINFO, SERVERNAME_A);
506
507 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
508 State.SPLITTING, System.currentTimeMillis(), SERVERNAME_A));
509 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
510
511 RegionTransitionData data = new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
512 REGIONINFO.getRegionName(), SERVERNAME_A);
513 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
514
515 ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
516
517 try {
518 processServerShutdownHandler(ct, am, regionSplitDone, null);
519
520
521
522 if(regionSplitDone){
523 assertTrue("Region state of region in SPLITTING should be removed from rit.",
524 am.regionsInTransition.isEmpty());
525 }
526 else{
527 while (!am.assignInvoked) {
528 Thread.sleep(1);
529 }
530 assertTrue("Assign should be invoked.", am.assignInvoked);
531 }
532 } finally {
533 REGIONINFO.setOffline(false);
534 REGIONINFO.setSplit(false);
535 executor.shutdown();
536 am.shutdown();
537
538 ZKAssign.deleteAllNodes(this.watcher);
539 }
540 }
541
542 private void testCaseWithPartiallyDisabledState(TableState state, boolean opening)
543 throws KeeperException, IOException, NodeExistsException {
544
545
546 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
547
548
549 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
550 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
551
552 AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer,
553 executor);
554 if (opening) {
555 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
556 State.OPENING, System.currentTimeMillis(), SERVERNAME_A));
557 } else {
558
559 am.regionOnline(REGIONINFO, SERVERNAME_A);
560
561 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
562 State.PENDING_CLOSE, System.currentTimeMillis(), SERVERNAME_A));
563 }
564 if (state == TableState.DISABLING) {
565 am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
566 } else {
567 am.getZKTable().setDisabledTable(REGIONINFO.getTableNameAsString());
568 }
569 RegionTransitionData data = null;
570 if (opening) {
571 data =
572 new RegionTransitionData(EventType.RS_ZK_REGION_OPENING, REGIONINFO.getRegionName(),
573 SERVERNAME_A);
574
575 } else {
576 data =
577 new RegionTransitionData(EventType.M_ZK_REGION_CLOSING, REGIONINFO.getRegionName(),
578 SERVERNAME_A);
579 }
580 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
581
582 ZKUtil.createAndWatch(this.watcher, node, data.getBytes());
583
584 try {
585 processServerShutdownHandler(ct, am, false, null);
586
587
588 assertTrue("The znode should be deleted.",ZKUtil.checkExists(this.watcher, node) == -1);
589 assertTrue("Region state of region in pending close should be removed from rit.",
590 am.regionsInTransition.isEmpty());
591 } finally {
592 executor.shutdown();
593 am.shutdown();
594
595 ZKAssign.deleteAllNodes(this.watcher);
596 }
597 }
598
599 private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am,
600 boolean splitRegion, ServerName sn)
601 throws IOException {
602
603
604 this.watcher.registerListenerFirst(am);
605
606
607 HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
608
609
610 Result r = null;
611 if (sn == null) {
612 if (splitRegion) {
613 r = getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
614 } else {
615 r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
616 }
617 } else {
618 if (sn.equals(SERVERNAME_A)) {
619 r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
620 } else if (sn.equals(SERVERNAME_B)) {
621 r = new Result(new KeyValue[0]);
622 }
623 }
624
625 Mockito.when(implementation.openScanner((byte [])Mockito.any(), (Scan)Mockito.any())).
626 thenReturn(System.currentTimeMillis());
627
628 Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).
629 thenReturn(new Result [] {r}, (Result [])null);
630
631
632 HConnection connection =
633 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
634 implementation, SERVERNAME_B, REGIONINFO);
635
636
637
638 Mockito.when(ct.getConnection()).thenReturn(connection);
639 Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
640
641
642
643 DeadServer deadServers = new DeadServer();
644 deadServers.add(SERVERNAME_A);
645
646 MasterServices services = Mockito.mock(MasterServices.class);
647 Mockito.when(services.getAssignmentManager()).thenReturn(am);
648 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
649 ServerShutdownHandler handler = null;
650 if (sn != null) {
651 handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false);
652 } else {
653 handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false);
654 }
655 handler.process();
656
657 }
658
659
660
661
662
663
664
665
666 private Result getMetaTableRowResult(final HRegionInfo hri,
667 final ServerName sn)
668 throws IOException {
669
670
671 List<KeyValue> kvs = new ArrayList<KeyValue>();
672 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
673 HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
674 Writables.getBytes(hri)));
675 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
676 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
677 Bytes.toBytes(sn.getHostAndPort())));
678 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
679 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
680 Bytes.toBytes(sn.getStartcode())));
681 return new Result(kvs);
682 }
683
684
685
686
687
688
689
690
691 private Result getMetaTableRowResultAsSplitRegion(final HRegionInfo hri, final ServerName sn)
692 throws IOException {
693 hri.setOffline(true);
694 hri.setSplit(true);
695 return getMetaTableRowResult(hri, sn);
696 }
697
698
699
700
701
702
703
704 private ExecutorService startupMasterExecutor(final String name) {
705
706 ExecutorService executor = new ExecutorService(name);
707 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
708 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
709 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
710 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
711 return executor;
712 }
713
714 @Test
715 public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
716
717 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
718
719
720
721 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
722
723 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
724 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
725 .getConfiguration());
726
727 AssignmentManager am =
728 new AssignmentManager(this.server, this.serverManager, ct, balancer, null);
729 try {
730
731 unassign(am, SERVERNAME_A, hri);
732
733 ZKAssign.deleteClosingNode(this.watcher, hri);
734
735
736
737 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
738
739
740
741
742 unassign(am, SERVERNAME_A, hri);
743
744 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
745 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
746 assertTrue(am.isRegionInTransition(hri) == null);
747 } finally {
748 am.shutdown();
749 }
750 }
751
752
753
754
755
756
757
758 @Test(timeout = 60000)
759 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
760 throws IOException, KeeperException, InterruptedException, ServiceException {
761 final RecoverableZooKeeper recoverableZk = Mockito
762 .mock(RecoverableZooKeeper.class);
763 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
764 this.server, this.serverManager);
765 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
766 null) {
767 public RecoverableZooKeeper getRecoverableZooKeeper() {
768 return recoverableZk;
769 }
770 };
771 ((ZooKeeperWatcher) zkw).registerListener(am);
772 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
773 .getChildren("/hbase/unassigned", zkw);
774 am.setWatcher((ZooKeeperWatcher) zkw);
775 try {
776 am.processDeadServersAndRegionsInTransition();
777 fail("Expected to abort");
778 } catch (NullPointerException e) {
779 fail("Should not throw NPE");
780 } catch (RuntimeException e) {
781 assertEquals("Aborted", e.getLocalizedMessage());
782 }
783 }
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
802 final HRegionInfo region, final ServerName serverName)
803 throws KeeperException, IOException {
804 RegionTransitionData data =
805 new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
806 region.getRegionName(), serverName);
807
808 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
809 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
810 throw new IOException("Failed create of ephemeral " + node);
811 }
812
813
814 return transitionNodeSplitting(zkw, region, serverName, -1);
815 }
816
817
818
819 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
820 final HRegionInfo parent,
821 final ServerName serverName, final int version)
822 throws KeeperException, IOException {
823 return ZKAssign.transitionNode(zkw, parent, serverName,
824 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
825 }
826
827 private void unassign(final AssignmentManager am, final ServerName sn,
828 final HRegionInfo hri) {
829
830 am.regionOnline(hri, sn);
831
832 am.unassign(hri);
833 }
834
835
836
837
838
839
840
841
842
843
844 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
845 final ServerManager manager)
846 throws IOException, KeeperException {
847
848 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
849
850
851
852
853
854 HRegionInterface ri = Mockito.mock(HRegionInterface.class);
855
856 Result[] result = null;
857 if (enabling) {
858 result = new Result[2];
859 result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
860 result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
861 }
862 Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
863 Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
864 thenReturn(System.currentTimeMillis());
865 if (enabling) {
866 Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).thenReturn(result, result, result,
867 (Result[]) null);
868
869 Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
870 getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
871 } else {
872
873 Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt())).thenReturn(new Result[] { r });
874
875 Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
876 }
877
878 HConnection connection = HConnectionTestingUtility.
879 getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
880 REGIONINFO);
881
882 Mockito.when(ct.getConnection()).thenReturn(connection);
883
884 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
885 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
886 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
887 server, manager, ct, balancer, executor);
888 return am;
889 }
890
891
892
893
894
895 @Test
896 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
897 ServiceException, InterruptedException {
898 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
899 MockedLoadBalancer.class, LoadBalancer.class);
900 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server,
901 this.serverManager);
902 try {
903
904
905 AtomicBoolean gate = new AtomicBoolean(false);
906 if (balancer instanceof MockedLoadBalancer) {
907 ((MockedLoadBalancer) balancer).setGateVariable(gate);
908 }
909 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
910 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
911 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
912 EventType.RS_ZK_REGION_FAILED_OPEN, v);
913 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
914 RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(),
915 SERVERNAME_A);
916 am.regionsInTransition.put(REGIONINFO.getEncodedName(), state);
917
918 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
919 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
920 List<ServerName> serverList = new ArrayList<ServerName>(2);
921 serverList.add(SERVERNAME_B);
922 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList);
923 am.nodeDataChanged(path);
924
925 while (!gate.get()) {
926 Thread.sleep(10);
927 }
928
929
930 RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
931 while (newRegionPlan == null) {
932 Thread.sleep(10);
933 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
934 }
935
936
937 assertNotSame("Same region plan should not come", regionPlan, newRegionPlan);
938 assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals(
939 newRegionPlan.getDestination())));
940 Mocking.waitForRegionOfflineInRIT(am, REGIONINFO.getEncodedName());
941 } finally {
942 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
943 DefaultLoadBalancer.class, LoadBalancer.class);
944 am.shutdown();
945 }
946 }
947
948
949
950
951
952
953
954
955
956 @Test
957 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
958 throws KeeperException, IOException, Exception {
959 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
960 MockedLoadBalancer.class, LoadBalancer.class);
961 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
962 new HashMap<ServerName, HServerLoad>(0));
963 List<ServerName> destServers = new ArrayList<ServerName>(1);
964 destServers.add(SERVERNAME_A);
965 Mockito.when(this.serverManager.getDrainingServersList()).thenReturn(destServers);
966
967
968 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
969 this.serverManager);
970 AtomicBoolean gate = new AtomicBoolean(false);
971 if (balancer instanceof MockedLoadBalancer) {
972 ((MockedLoadBalancer) balancer).setGateVariable(gate);
973 }
974 try{
975
976 am.getZKTable().setDisablingTable(REGIONINFO.getTableNameAsString());
977 am.joinCluster();
978
979 assertFalse(
980 "Assign should not be invoked for disabling table regions during clean cluster startup.",
981 gate.get());
982
983 assertTrue("Table should be disabled.",
984 am.getZKTable().isDisabledTable(REGIONINFO.getTableNameAsString()));
985 } finally {
986 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
987 DefaultLoadBalancer.class, LoadBalancer.class);
988 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
989 am.shutdown();
990 }
991 }
992
993
994
995
996
997
998
999
1000 @Test
1001 public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
1002 throws KeeperException, IOException, Exception {
1003 List<ServerName> destServers = new ArrayList<ServerName>(1);
1004 destServers.add(SERVERNAME_A);
1005 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(destServers);
1006 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1007 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1008 Server server = new HMaster(HTU.getConfiguration());
1009 Whitebox.setInternalState(server, "serverManager", this.serverManager);
1010 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1011 this.serverManager);
1012 try {
1013 String tableName = "dummyTable";
1014 am.enablingTables.put(tableName, null);
1015
1016 am.getZKTable().setEnablingTable(tableName);
1017 am.joinCluster();
1018 assertFalse("Table should not be present in zookeeper.",
1019 am.getZKTable().isTablePresent(tableName));
1020 } finally {
1021 }
1022 }
1023
1024
1025
1026
1027
1028
1029
1030
1031 @Test
1032 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
1033 enabling = true;
1034 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
1035 DefaultLoadBalancer.class, LoadBalancer.class);
1036 Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
1037 serverAndLoad.put(SERVERNAME_A, null);
1038 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
1039 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
1040 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1041 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1042 Server server = new HMaster(HTU.getConfiguration());
1043 Whitebox.setInternalState(server, "serverManager", this.serverManager);
1044 assignmentCount = 0;
1045 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1046 this.serverManager);
1047 am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
1048 HConstants.EMPTY_END_ROW), SERVERNAME_A);
1049 am.gate.set(false);
1050 try {
1051
1052 am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
1053 ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
1054
1055 am.joinCluster();
1056 while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
1057 Thread.sleep(10);
1058 }
1059 assertEquals("Number of assignments should be equal.", 2, assignmentCount);
1060 assertTrue("Table should be enabled.",
1061 am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
1062 } finally {
1063 enabling = false;
1064 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1065 am.shutdown();
1066 ZKAssign.deleteAllNodes(this.watcher);
1067 assignmentCount = 0;
1068 }
1069 }
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082 @Test
1083 public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException,
1084 ServiceException {
1085 testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(true);
1086 testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(false);
1087 }
1088
1089 private void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown(boolean regionInOffline)
1090 throws IOException, KeeperException, ServiceException {
1091
1092 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1093
1094 AssignmentManagerWithExtrasForTesting am =
1095 setUpMockedAssignmentManager(this.server, this.serverManager);
1096
1097 if (regionInOffline) {
1098 ServerName MASTER_SERVERNAME = new ServerName("example.org", 1111, 1111);
1099 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1100 State.OFFLINE, System.currentTimeMillis(), MASTER_SERVERNAME));
1101 } else {
1102 am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO,
1103 State.OPENING, System.currentTimeMillis(), SERVERNAME_B));
1104 }
1105
1106 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B));
1107 am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
1108
1109 try {
1110 processServerShutdownHandler(ct, am, false, SERVERNAME_A);
1111 processServerShutdownHandler(ct, am, false, SERVERNAME_B);
1112 if(regionInOffline){
1113 assertFalse("Assign should not be invoked.", am.assignInvoked);
1114 } else {
1115 assertTrue("Assign should be invoked.", am.assignInvoked);
1116 }
1117 } finally {
1118 am.regionsInTransition.remove(REGIONINFO.getEncodedName());
1119 am.regionPlans.remove(REGIONINFO.getEncodedName());
1120 }
1121 }
1122
1123
1124
1125
1126
1127 public static class MockedLoadBalancer extends DefaultLoadBalancer {
1128 private AtomicBoolean gate;
1129
1130 public void setGateVariable(AtomicBoolean gate) {
1131 this.gate = gate;
1132 }
1133
1134 @Override
1135 public ServerName randomAssignment(List<ServerName> servers) {
1136 ServerName randomServerName = super.randomAssignment(servers);
1137 this.gate.set(true);
1138 return randomServerName;
1139 }
1140
1141 @Override
1142 public Map<ServerName, List<HRegionInfo>> retainAssignment(
1143 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
1144 this.gate.set(true);
1145 return super.retainAssignment(regions, servers);
1146 }
1147
1148 }
1149
1150
1151
1152
1153 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1154
1155 private final ExecutorService es;
1156
1157 private final CatalogTracker ct;
1158 boolean processRITInvoked = false;
1159 boolean assignInvoked = false;
1160 AtomicBoolean gate = new AtomicBoolean(true);
1161
1162 public AssignmentManagerWithExtrasForTesting(final Server master,
1163 final ServerManager serverManager, final CatalogTracker catalogTracker,
1164 final LoadBalancer balancer, final ExecutorService service)
1165 throws KeeperException, IOException {
1166 super(master, serverManager, catalogTracker, balancer, service);
1167 this.es = service;
1168 this.ct = catalogTracker;
1169 }
1170
1171 @Override
1172 boolean processRegionInTransition(String encodedRegionName,
1173 HRegionInfo regionInfo,
1174 Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers)
1175 throws KeeperException, IOException {
1176 this.processRITInvoked = true;
1177 return super.processRegionInTransition(encodedRegionName, regionInfo,
1178 deadServers);
1179 }
1180 @Override
1181 void processRegionsInTransition(final RegionTransitionData data,
1182 final HRegionInfo regionInfo,
1183 final Map<ServerName, List<Pair<HRegionInfo, Result>>> deadServers,
1184 final int expectedVersion) throws KeeperException {
1185 while (this.gate.get()) Threads.sleep(1);
1186 super.processRegionsInTransition(data, regionInfo, deadServers, expectedVersion);
1187 }
1188
1189 @Override
1190 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
1191 boolean hijack) {
1192 if (enabling) {
1193 assignmentCount++;
1194 this.regionOnline(region, SERVERNAME_A);
1195 } else {
1196 assignInvoked = true;
1197 super.assign(region, setOfflineInZK, forceNewPlan, hijack);
1198 }
1199 }
1200
1201 @Override
1202 public ServerName getRegionServerOfRegion(HRegionInfo hri) {
1203 return SERVERNAME_A;
1204 }
1205
1206
1207 void setWatcher(ZooKeeperWatcher watcher) {
1208 this.watcher = watcher;
1209 }
1210
1211
1212
1213
1214 ExecutorService getExecutorService() {
1215 return this.es;
1216 }
1217
1218
1219
1220
1221 CatalogTracker getCatalogTracker() {
1222 return this.ct;
1223 }
1224 }
1225
1226
1227
1228
1229
1230
1231
1232 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1233 final ZooKeeperWatcher watcher) {
1234
1235
1236 watcher.registerListenerFirst(am);
1237 Thread t = new Thread("RunAmJoinCluster") {
1238 public void run() {
1239
1240
1241
1242
1243
1244 am.regionsInTransition.clear();
1245 am.regionPlans.clear();
1246 try {
1247 am.joinCluster();
1248 } catch (IOException e) {
1249 throw new RuntimeException(e);
1250 } catch (KeeperException e) {
1251 throw new RuntimeException(e);
1252 } catch (InterruptedException e) {
1253 throw new RuntimeException(e);
1254 }
1255 };
1256 };
1257 t.start();
1258 while (!t.isAlive()) Threads.sleep(1);
1259 }
1260
1261 @org.junit.Rule
1262 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1263 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1264 }