1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.hbase.CellScannable;
34 import org.apache.hadoop.hbase.CellUtil;
35 import org.apache.hadoop.hbase.HBaseConfiguration;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.MediumTests;
40 import org.apache.hadoop.hbase.RegionException;
41 import org.apache.hadoop.hbase.RegionTransition;
42 import org.apache.hadoop.hbase.Server;
43 import org.apache.hadoop.hbase.ServerLoad;
44 import org.apache.hadoop.hbase.ServerName;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
47 import org.apache.hadoop.hbase.catalog.CatalogTracker;
48 import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
49 import org.apache.hadoop.hbase.client.HConnection;
50 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
51 import org.apache.hadoop.hbase.client.Result;
52 import org.apache.hadoop.hbase.exceptions.DeserializationException;
53 import org.apache.hadoop.hbase.executor.EventType;
54 import org.apache.hadoop.hbase.executor.ExecutorService;
55 import org.apache.hadoop.hbase.executor.ExecutorType;
56 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
57 import org.apache.hadoop.hbase.master.RegionState.State;
58 import org.apache.hadoop.hbase.master.TableLockManager.NullTableLockManager;
59 import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
60 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
61 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
62 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
63 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
64 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
65 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
66 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
67 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
68 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
69 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table;
70 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
71 import org.apache.hadoop.hbase.util.Bytes;
72 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73 import org.apache.hadoop.hbase.util.Threads;
74 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
75 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
76 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
77 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
78 import org.apache.zookeeper.KeeperException;
79 import org.apache.zookeeper.KeeperException.NodeExistsException;
80 import org.apache.zookeeper.Watcher;
81 import org.junit.After;
82 import org.junit.AfterClass;
83 import org.junit.Before;
84 import org.junit.BeforeClass;
85 import org.junit.Test;
86 import org.junit.experimental.categories.Category;
87 import org.mockito.Mockito;
88 import org.mockito.internal.util.reflection.Whitebox;
89 import org.mockito.invocation.InvocationOnMock;
90 import org.mockito.stubbing.Answer;
91
92 import com.google.protobuf.RpcController;
93 import com.google.protobuf.ServiceException;
94
95
96
97
98
99 @Category(MediumTests.class)
100 public class TestAssignmentManager {
101 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
102 private static final ServerName SERVERNAME_A =
103 ServerName.valueOf("example.org", 1234, 5678);
104 private static final ServerName SERVERNAME_B =
105 ServerName.valueOf("example.org", 0, 5678);
106 private static final HRegionInfo REGIONINFO =
107 new HRegionInfo(TableName.valueOf("t"),
108 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
109 private static int assignmentCount;
110 private static boolean enabling = false;
111
112
113 private Server server;
114 private ServerManager serverManager;
115 private ZooKeeperWatcher watcher;
116 private LoadBalancer balancer;
117 private HMaster master;
118
119 @BeforeClass
120 public static void beforeClass() throws Exception {
121 HTU.startMiniZKCluster();
122 }
123
124 @AfterClass
125 public static void afterClass() throws IOException {
126 HTU.shutdownMiniZKCluster();
127 }
128
129 @Before
130 public void before() throws ZooKeeperConnectionException, IOException {
131
132
133
134
135
136
137 this.server = Mockito.mock(Server.class);
138 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
139 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
140 this.watcher =
141 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
142 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
143 Mockito.doThrow(new RuntimeException("Aborted")).
144 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
145
146
147
148 this.serverManager = Mockito.mock(ServerManager.class);
149 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
150 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
151 Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer());
152 final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
153 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
154 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
155 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
156 new ArrayList<ServerName>(onlineServers.keySet()));
157 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
158
159 List<ServerName> avServers = new ArrayList<ServerName>();
160 avServers.addAll(onlineServers.keySet());
161 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
162 Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers);
163
164 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
165 thenReturn(true);
166 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
167 thenReturn(true);
168
169 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)).
170 thenReturn(RegionOpeningState.OPENED);
171 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)).
172 thenReturn(RegionOpeningState.OPENED);
173 this.master = Mockito.mock(HMaster.class);
174
175 Mockito.when(this.master.getServerManager()).thenReturn(serverManager);
176 }
177
178 @After
179 public void after() throws KeeperException {
180 if (this.watcher != null) {
181
182 ZKAssign.deleteAllNodes(this.watcher);
183 this.watcher.close();
184 }
185 }
186
187
188
189
190
191
192
193
194
195 @Test(timeout = 60000)
196 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
197 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
198 AssignmentManagerWithExtrasForTesting am =
199 setUpMockedAssignmentManager(this.server, this.serverManager);
200 try {
201 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
202 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
203 while (!am.processRITInvoked) Thread.sleep(1);
204
205
206
207 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
208
209 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
210
211
212
213
214
215
216 int versionid =
217 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
218 assertNotSame(versionid, -1);
219 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
220
221
222
223 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
224 assertNotSame(-1, versionid);
225
226 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
227 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
228 EventType.RS_ZK_REGION_OPENING, versionid);
229 assertNotSame(-1, versionid);
230
231 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
232 SERVERNAME_B, versionid);
233 assertNotSame(-1, versionid);
234 am.gate.set(false);
235
236 ZKAssign.blockUntilNoRIT(watcher);
237 } finally {
238 am.getExecutorService().shutdown();
239 am.shutdown();
240 }
241 }
242
243 @Test(timeout = 60000)
244 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
245 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
246 AssignmentManagerWithExtrasForTesting am =
247 setUpMockedAssignmentManager(this.server, this.serverManager);
248 try {
249 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
250 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
251 while (!am.processRITInvoked) Thread.sleep(1);
252
253
254
255 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
256
257 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
258
259
260
261
262
263
264 int versionid =
265 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
266 assertNotSame(versionid, -1);
267 am.gate.set(false);
268 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
269
270
271
272 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
273 assertNotSame(-1, versionid);
274
275 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
276 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
277 EventType.RS_ZK_REGION_OPENING, versionid);
278 assertNotSame(-1, versionid);
279
280 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
281 SERVERNAME_B, versionid);
282 assertNotSame(-1, versionid);
283
284
285 ZKAssign.blockUntilNoRIT(watcher);
286 } finally {
287 am.getExecutorService().shutdown();
288 am.shutdown();
289 }
290 }
291
292 @Test(timeout = 60000)
293 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
294 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
295 AssignmentManagerWithExtrasForTesting am =
296 setUpMockedAssignmentManager(this.server, this.serverManager);
297 try {
298 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
299 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
300 while (!am.processRITInvoked) Thread.sleep(1);
301
302
303
304 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
305
306 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
307
308
309
310
311
312
313 int versionid =
314 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
315 assertNotSame(versionid, -1);
316 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
317
318 am.gate.set(false);
319
320
321 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
322 assertNotSame(-1, versionid);
323
324 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
325 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
326 EventType.RS_ZK_REGION_OPENING, versionid);
327 assertNotSame(-1, versionid);
328
329 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
330 SERVERNAME_B, versionid);
331 assertNotSame(-1, versionid);
332
333 ZKAssign.blockUntilNoRIT(watcher);
334 } finally {
335 am.getExecutorService().shutdown();
336 am.shutdown();
337 }
338 }
339
340 private void createRegionPlanAndBalance(
341 final AssignmentManager am, final ServerName from,
342 final ServerName to, final HRegionInfo hri) throws RegionException {
343
344
345 am.regionOnline(hri, from);
346
347
348 am.balance(new RegionPlan(hri, from, to));
349 }
350
351
352
353
354
355
356
357
358 @Test
359 public void testBalance()
360 throws IOException, KeeperException, DeserializationException, InterruptedException {
361
362
363 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
364
365
366 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
367 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
368 .getConfiguration());
369
370 AssignmentManager am = new AssignmentManager(this.server,
371 this.serverManager, ct, balancer, executor, null, master.getTableLockManager());
372 am.failoverCleanupDone.set(true);
373 try {
374
375
376 this.watcher.registerListenerFirst(am);
377
378
379 am.regionOnline(REGIONINFO, SERVERNAME_A);
380
381 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
382 am.balance(plan);
383
384 RegionStates regionStates = am.getRegionStates();
385
386 assertTrue(regionStates.isRegionInTransition(REGIONINFO)
387 && regionStates.isRegionInState(REGIONINFO, State.FAILED_CLOSE));
388
389 regionStates.updateRegionState(REGIONINFO, State.PENDING_CLOSE);
390
391
392
393
394
395
396 int versionid =
397 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
398 assertNotSame(versionid, -1);
399
400
401
402
403
404 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
405
406
407 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
408 assertNotSame(-1, versionid);
409
410 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
411 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
412 EventType.RS_ZK_REGION_OPENING, versionid);
413 assertNotSame(-1, versionid);
414
415 versionid =
416 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
417 assertNotSame(-1, versionid);
418
419 while(regionStates.isRegionInTransition(REGIONINFO)) Threads.sleep(1);
420 } finally {
421 executor.shutdown();
422 am.shutdown();
423
424 ZKAssign.deleteAllNodes(this.watcher);
425 }
426 }
427
428
429
430
431
432
433 @Test
434 public void testShutdownHandler()
435 throws KeeperException, IOException, ServiceException {
436
437
438 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
439
440
441 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
442
443 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
444 this.server, this.serverManager);
445 try {
446 processServerShutdownHandler(ct, am, false);
447 } finally {
448 executor.shutdown();
449 am.shutdown();
450
451 ZKAssign.deleteAllNodes(this.watcher);
452 }
453 }
454
455
456
457
458
459
460
461
462
463
464 @Test
465 public void testSSHWhenDisableTableInProgress() throws KeeperException, IOException,
466 ServiceException {
467 testCaseWithPartiallyDisabledState(Table.State.DISABLING);
468 testCaseWithPartiallyDisabledState(Table.State.DISABLED);
469 }
470
471
472
473
474
475
476
477
478
479 @Test
480 public void testSSHWhenSplitRegionInProgress() throws KeeperException, IOException, Exception {
481
482 testCaseWithSplitRegionPartial(true);
483
484 testCaseWithSplitRegionPartial(false);
485 }
486
487 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException,
488 IOException, NodeExistsException, InterruptedException, ServiceException {
489
490
491 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
492
493 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
494 ZKAssign.deleteAllNodes(this.watcher);
495
496
497 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
498 this.server, this.serverManager);
499
500 am.regionOnline(REGIONINFO, SERVERNAME_A);
501
502 am.getRegionStates().updateRegionState(
503 REGIONINFO, State.SPLITTING, SERVERNAME_A);
504 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
505 RegionTransition data = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
506 REGIONINFO.getRegionName(), SERVERNAME_A);
507 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
508
509 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
510
511 try {
512 processServerShutdownHandler(ct, am, regionSplitDone);
513
514
515
516 if (regionSplitDone) {
517 assertFalse("Region state of region in SPLITTING should be removed from rit.",
518 am.getRegionStates().isRegionsInTransition());
519 } else {
520 while (!am.assignInvoked) {
521 Thread.sleep(1);
522 }
523 assertTrue("Assign should be invoked.", am.assignInvoked);
524 }
525 } finally {
526 REGIONINFO.setOffline(false);
527 REGIONINFO.setSplit(false);
528 executor.shutdown();
529 am.shutdown();
530
531 ZKAssign.deleteAllNodes(this.watcher);
532 }
533 }
534
535 private void testCaseWithPartiallyDisabledState(Table.State state) throws KeeperException,
536 IOException, NodeExistsException, ServiceException {
537
538
539 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
540
541 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
542 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
543 ZKAssign.deleteAllNodes(this.watcher);
544
545
546 AssignmentManager am = new AssignmentManager(this.server,
547 this.serverManager, ct, balancer, executor, null, master.getTableLockManager());
548
549 am.regionOnline(REGIONINFO, SERVERNAME_A);
550
551 am.getRegionStates().updateRegionState(REGIONINFO, State.PENDING_CLOSE);
552 if (state == Table.State.DISABLING) {
553 am.getZKTable().setDisablingTable(REGIONINFO.getTable());
554 } else {
555 am.getZKTable().setDisabledTable(REGIONINFO.getTable());
556 }
557 RegionTransition data = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING,
558 REGIONINFO.getRegionName(), SERVERNAME_A);
559
560
561
562 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
563
564 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
565
566 try {
567 processServerShutdownHandler(ct, am, false);
568
569
570 assertTrue("The znode should be deleted.", ZKUtil.checkExists(this.watcher, node) == -1);
571
572
573
574 if (state == Table.State.DISABLED) {
575 assertFalse("Region state of region in pending close should be removed from rit.",
576 am.getRegionStates().isRegionsInTransition());
577 }
578 } finally {
579 am.setEnabledTable(REGIONINFO.getTable());
580 executor.shutdown();
581 am.shutdown();
582
583 ZKAssign.deleteAllNodes(this.watcher);
584 }
585 }
586
587 private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am, boolean splitRegion)
588 throws IOException, ServiceException {
589
590
591 this.watcher.registerListenerFirst(am);
592
593
594
595 ClientProtos.ClientService.BlockingInterface implementation =
596 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
597
598
599 Result r;
600 if (splitRegion) {
601 r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
602 } else {
603 r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
604 }
605
606 final ScanResponse.Builder builder = ScanResponse.newBuilder();
607 builder.setMoreResults(true);
608 builder.addCellsPerResult(r.size());
609 final List<CellScannable> cellScannables = new ArrayList<CellScannable>(1);
610 cellScannables.add(r);
611 Mockito.when(implementation.scan(
612 (RpcController)Mockito.any(), (ScanRequest)Mockito.any())).
613 thenAnswer(new Answer<ScanResponse>() {
614 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
615 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
616 .getArguments()[0];
617 if (controller != null) {
618 controller.setCellScanner(CellUtil.createCellScanner(cellScannables));
619 }
620 return builder.build();
621 }
622 });
623
624
625 HConnection connection =
626 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
627 null, implementation, SERVERNAME_B, REGIONINFO);
628
629
630
631 Mockito.when(ct.getConnection()).thenReturn(connection);
632 Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
633
634
635
636 DeadServer deadServers = new DeadServer();
637 deadServers.add(SERVERNAME_A);
638
639 MasterServices services = Mockito.mock(MasterServices.class);
640 Mockito.when(services.getAssignmentManager()).thenReturn(am);
641 Mockito.when(services.getServerManager()).thenReturn(this.serverManager);
642 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
643 ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
644 services, deadServers, SERVERNAME_A, false);
645 am.failoverCleanupDone.set(true);
646 handler.process();
647
648 }
649
650
651
652
653
654
655
656 private ExecutorService startupMasterExecutor(final String name) {
657
658 ExecutorService executor = new ExecutorService(name);
659 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
660 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
661 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
662 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
663 return executor;
664 }
665
666 @Test
667 public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
668
669 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
670
671
672
673 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
674
675 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
676 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
677 .getConfiguration());
678
679 AssignmentManager am = new AssignmentManager(this.server,
680 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
681 try {
682
683 unassign(am, SERVERNAME_A, hri);
684
685 ZKAssign.deleteClosingNode(this.watcher, hri, SERVERNAME_A);
686
687
688
689 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
690
691
692
693
694 unassign(am, SERVERNAME_A, hri);
695
696 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
697 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
698 assertFalse(am.getRegionStates().isRegionInTransition(hri));
699 } finally {
700 am.shutdown();
701 }
702 }
703
704
705
706
707
708
709
710 @Test(timeout = 60000)
711 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
712 throws IOException, KeeperException, InterruptedException, ServiceException {
713 final RecoverableZooKeeper recoverableZk = Mockito
714 .mock(RecoverableZooKeeper.class);
715 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
716 this.server, this.serverManager);
717 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
718 null) {
719 public RecoverableZooKeeper getRecoverableZooKeeper() {
720 return recoverableZk;
721 }
722 };
723 ((ZooKeeperWatcher) zkw).registerListener(am);
724 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
725 .getChildren("/hbase/region-in-transition", null);
726 am.setWatcher((ZooKeeperWatcher) zkw);
727 try {
728 am.processDeadServersAndRegionsInTransition(null);
729 fail("Expected to abort");
730 } catch (NullPointerException e) {
731 fail("Should not throw NPE");
732 } catch (RuntimeException e) {
733 assertEquals("Aborted", e.getLocalizedMessage());
734 }
735 }
736
737
738
739
740 @Test(timeout = 60000)
741 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
742 ServiceException, InterruptedException {
743 this.server.getConfiguration().setClass(
744 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockedLoadBalancer.class,
745 LoadBalancer.class);
746 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
747 this.server, this.serverManager);
748 try {
749
750
751
752 AtomicBoolean gate = new AtomicBoolean(false);
753 if (balancer instanceof MockedLoadBalancer) {
754 ((MockedLoadBalancer) balancer).setGateVariable(gate);
755 }
756 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
757 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
758 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A,
759 EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
760 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO
761 .getEncodedName());
762 am.getRegionStates().updateRegionState(
763 REGIONINFO, State.OPENING, SERVERNAME_A);
764
765
766 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(
767 REGIONINFO, null, SERVERNAME_A));
768 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
769 List<ServerName> serverList = new ArrayList<ServerName>(2);
770 serverList.add(SERVERNAME_B);
771 Mockito.when(
772 this.serverManager.createDestinationServersList(SERVERNAME_A))
773 .thenReturn(serverList);
774 am.nodeDataChanged(path);
775
776
777 while (!gate.get()) {
778 Thread.sleep(10);
779 }
780
781
782
783 RegionPlan newRegionPlan = am.regionPlans
784 .get(REGIONINFO.getEncodedName());
785 while (newRegionPlan == null) {
786 Thread.sleep(10);
787 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
788 }
789
790
791
792 assertNotSame("Same region plan should not come", regionPlan,
793 newRegionPlan);
794 assertTrue("Destination servers should be different.", !(regionPlan
795 .getDestination().equals(newRegionPlan.getDestination())));
796
797 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
798 } finally {
799 this.server.getConfiguration().setClass(
800 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SimpleLoadBalancer.class,
801 LoadBalancer.class);
802 am.getExecutorService().shutdown();
803 am.shutdown();
804 }
805 }
806
807
808
809
810
811 public static class MockedLoadBalancer extends SimpleLoadBalancer {
812 private AtomicBoolean gate;
813
814 public void setGateVariable(AtomicBoolean gate) {
815 this.gate = gate;
816 }
817
818 @Override
819 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
820 ServerName randomServerName = super.randomAssignment(regionInfo, servers);
821 this.gate.set(true);
822 return randomServerName;
823 }
824
825 @Override
826 public Map<ServerName, List<HRegionInfo>> retainAssignment(
827 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
828 this.gate.set(true);
829 return super.retainAssignment(regions, servers);
830 }
831 }
832
833
834
835
836
837
838 @Test(timeout = 60000)
839 public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException,
840 KeeperException, ServiceException, InterruptedException {
841 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
842 this.server, this.serverManager);
843 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
844 int version = ZKAssign.getVersion(this.watcher, REGIONINFO);
845 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
846 EventType.RS_ZK_REGION_OPENING, version);
847 RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING,
848 REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY);
849 version = ZKAssign.getVersion(this.watcher, REGIONINFO);
850 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
851 am.getRegionStates().logSplit(SERVERNAME_A);
852 am.getRegionStates().createRegionState(REGIONINFO);
853 am.gate.set(false);
854 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
855 assertFalse(am.processRegionsInTransition(rt, REGIONINFO, version));
856 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
857 processServerShutdownHandler(ct, am, false);
858
859 while (!am.gate.get()) {
860 Thread.sleep(10);
861 }
862 assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO
863 .getEncodedName()));
864 }
865
866
867
868
869
870
871
872
873
874 @Test(timeout = 60000)
875 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
876 throws KeeperException, IOException, Exception {
877 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
878 MockedLoadBalancer.class, LoadBalancer.class);
879 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
880 new HashMap<ServerName, ServerLoad>(0));
881 List<ServerName> destServers = new ArrayList<ServerName>(1);
882 destServers.add(SERVERNAME_A);
883 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
884
885 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
886 Server server = new HMaster(HTU.getConfiguration());
887 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
888 this.serverManager);
889 AtomicBoolean gate = new AtomicBoolean(false);
890 if (balancer instanceof MockedLoadBalancer) {
891 ((MockedLoadBalancer) balancer).setGateVariable(gate);
892 }
893 try{
894
895 am.getZKTable().setDisablingTable(REGIONINFO.getTable());
896 am.joinCluster();
897
898 assertFalse(
899 "Assign should not be invoked for disabling table regions during clean cluster startup.",
900 gate.get());
901
902 assertTrue("Table should be disabled.",
903 am.getZKTable().isDisabledTable(REGIONINFO.getTable()));
904 } finally {
905 this.server.getConfiguration().setClass(
906 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SimpleLoadBalancer.class,
907 LoadBalancer.class);
908 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
909 am.shutdown();
910 }
911 }
912
913
914
915
916
917
918
919
920 @Test
921 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
922 enabling = true;
923 List<ServerName> destServers = new ArrayList<ServerName>(1);
924 destServers.add(SERVERNAME_A);
925 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
926 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
927 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
928 Server server = new HMaster(HTU.getConfiguration());
929 Whitebox.setInternalState(server, "serverManager", this.serverManager);
930 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
931 this.serverManager);
932 try {
933
934 am.getZKTable().setEnablingTable(REGIONINFO.getTable());
935 new EnableTableHandler(server, REGIONINFO.getTable(),
936 am.getCatalogTracker(), am, new NullTableLockManager(), true).prepare()
937 .process();
938 assertEquals("Number of assignments should be 1.", 1, assignmentCount);
939 assertTrue("Table should be enabled.",
940 am.getZKTable().isEnabledTable(REGIONINFO.getTable()));
941 } finally {
942 enabling = false;
943 assignmentCount = 0;
944 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
945 am.shutdown();
946 ZKAssign.deleteAllNodes(this.watcher);
947 }
948 }
949
950
951
952
953
954
955 @Test
956 public void testSSHTimesOutOpeningRegionTransition()
957 throws KeeperException, IOException, ServiceException {
958
959 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
960
961 AssignmentManagerWithExtrasForTesting am =
962 setUpMockedAssignmentManager(this.server, this.serverManager);
963
964 RegionState state = new RegionState(REGIONINFO,
965 State.OPENING, System.currentTimeMillis(), SERVERNAME_A);
966 am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_B);
967 am.getRegionStates().regionsInTransition.put(REGIONINFO.getEncodedName(), state);
968
969 am.regionPlans.put(REGIONINFO.getEncodedName(),
970 new RegionPlan(REGIONINFO, SERVERNAME_B, SERVERNAME_A));
971 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
972
973 try {
974 am.assignInvoked = false;
975 processServerShutdownHandler(ct, am, false);
976 assertTrue(am.assignInvoked);
977 } finally {
978 am.getRegionStates().regionsInTransition.remove(REGIONINFO.getEncodedName());
979 am.regionPlans.remove(REGIONINFO.getEncodedName());
980 }
981 }
982
983
984
985
986
987
988
989
990
991
992 @Test
993 public void testClosingFailureDuringRecovery() throws Exception {
994
995 AssignmentManagerWithExtrasForTesting am =
996 setUpMockedAssignmentManager(this.server, this.serverManager);
997 ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_A);
998 am.getRegionStates().createRegionState(REGIONINFO);
999
1000 assertFalse( am.getRegionStates().isRegionsInTransition() );
1001
1002 am.processRegionInTransition(REGIONINFO.getEncodedName(), REGIONINFO);
1003
1004 assertTrue( am.getRegionStates().isRegionsInTransition() );
1005 }
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
1024 final HRegionInfo region, final ServerName serverName)
1025 throws KeeperException, IOException {
1026 RegionTransition rt =
1027 RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
1028 region.getRegionName(), serverName);
1029
1030 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1031 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1032 throw new IOException("Failed create of ephemeral " + node);
1033 }
1034
1035
1036 return transitionNodeSplitting(zkw, region, serverName, -1);
1037 }
1038
1039
1040
1041 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
1042 final HRegionInfo parent,
1043 final ServerName serverName, final int version)
1044 throws KeeperException, IOException {
1045 return ZKAssign.transitionNode(zkw, parent, serverName,
1046 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
1047 }
1048
1049 private void unassign(final AssignmentManager am, final ServerName sn,
1050 final HRegionInfo hri) throws RegionException {
1051
1052 am.regionOnline(hri, sn);
1053
1054 am.unassign(hri);
1055 }
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
1067 final ServerManager manager) throws IOException, KeeperException, ServiceException {
1068
1069 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1070
1071
1072
1073
1074
1075 ClientProtos.ClientService.BlockingInterface ri =
1076 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
1077
1078 Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
1079 final ScanResponse.Builder builder = ScanResponse.newBuilder();
1080 builder.setMoreResults(true);
1081 builder.addCellsPerResult(r.size());
1082 final List<CellScannable> rows = new ArrayList<CellScannable>(1);
1083 rows.add(r);
1084 Answer<ScanResponse> ans = new Answer<ClientProtos.ScanResponse>() {
1085 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
1086 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
1087 .getArguments()[0];
1088 if (controller != null) {
1089 controller.setCellScanner(CellUtil.createCellScanner(rows));
1090 }
1091 return builder.build();
1092 }
1093 };
1094 if (enabling) {
1095 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any()))
1096 .thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans)
1097 .thenReturn(ScanResponse.newBuilder().setMoreResults(false).build());
1098 } else {
1099 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any())).thenAnswer(
1100 ans);
1101 }
1102
1103 GetResponse.Builder getBuilder = GetResponse.newBuilder();
1104 getBuilder.setResult(ProtobufUtil.toResult(r));
1105 Mockito.when(ri.get((RpcController)Mockito.any(), (GetRequest) Mockito.any())).
1106 thenReturn(getBuilder.build());
1107
1108 HConnection connection = HConnectionTestingUtility.
1109 getMockedConnectionAndDecorate(HTU.getConfiguration(), null,
1110 ri, SERVERNAME_B, REGIONINFO);
1111
1112 Mockito.when(ct.getConnection()).thenReturn(connection);
1113
1114 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
1115 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
1116 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
1117 server, manager, ct, this.balancer, executor, new NullTableLockManager());
1118 return am;
1119 }
1120
1121
1122
1123
1124 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1125
1126 private final ExecutorService es;
1127
1128 private final CatalogTracker ct;
1129 boolean processRITInvoked = false;
1130 boolean assignInvoked = false;
1131 AtomicBoolean gate = new AtomicBoolean(true);
1132
1133 public AssignmentManagerWithExtrasForTesting(
1134 final Server master, final ServerManager serverManager,
1135 final CatalogTracker catalogTracker, final LoadBalancer balancer,
1136 final ExecutorService service, final TableLockManager tableLockManager)
1137 throws KeeperException, IOException {
1138 super(master, serverManager, catalogTracker, balancer, service, null, tableLockManager);
1139 this.es = service;
1140 this.ct = catalogTracker;
1141 }
1142
1143 @Override
1144 boolean processRegionInTransition(String encodedRegionName,
1145 HRegionInfo regionInfo) throws KeeperException, IOException {
1146 this.processRITInvoked = true;
1147 return super.processRegionInTransition(encodedRegionName, regionInfo);
1148 }
1149
1150 @Override
1151 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan) {
1152 if (enabling) {
1153 assignmentCount++;
1154 this.regionOnline(region, SERVERNAME_A);
1155 } else {
1156 super.assign(region, setOfflineInZK, forceNewPlan);
1157 this.gate.set(true);
1158 }
1159 }
1160
1161 @Override
1162 public void assign(List<HRegionInfo> regions)
1163 throws IOException, InterruptedException {
1164 assignInvoked = (regions != null && regions.size() > 0);
1165 super.assign(regions);
1166 this.gate.set(true);
1167 }
1168
1169
1170 void setWatcher(ZooKeeperWatcher watcher) {
1171 this.watcher = watcher;
1172 }
1173
1174
1175
1176
1177 ExecutorService getExecutorService() {
1178 return this.es;
1179 }
1180
1181
1182
1183
1184 CatalogTracker getCatalogTracker() {
1185 return this.ct;
1186 }
1187 }
1188
1189
1190
1191
1192
1193
1194
1195 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1196 final ZooKeeperWatcher watcher) {
1197
1198
1199 watcher.registerListenerFirst(am);
1200 Thread t = new Thread("RunAmJoinCluster") {
1201 public void run() {
1202
1203
1204
1205
1206
1207 am.getRegionStates().regionsInTransition.clear();
1208 am.regionPlans.clear();
1209 try {
1210 am.joinCluster();
1211 } catch (IOException e) {
1212 throw new RuntimeException(e);
1213 } catch (KeeperException e) {
1214 throw new RuntimeException(e);
1215 } catch (InterruptedException e) {
1216 throw new RuntimeException(e);
1217 }
1218 }
1219 };
1220 t.start();
1221 while (!t.isAlive()) Threads.sleep(1);
1222 }
1223
1224 @Test
1225 public void testForceAssignMergingRegion() throws Exception {
1226
1227 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1228
1229 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1230 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1231 server.getConfiguration());
1232
1233 AssignmentManager am = new AssignmentManager(this.server,
1234 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
1235 RegionStates regionStates = am.getRegionStates();
1236 try {
1237
1238 regionStates.updateRegionState(hri, RegionState.State.MERGING);
1239
1240 am.assign(hri, true, true);
1241 assertEquals("The region should be still in merging state",
1242 RegionState.State.MERGING, regionStates.getRegionState(hri).getState());
1243 } finally {
1244 am.shutdown();
1245 }
1246 }
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258 @Test
1259 public void testAssignmentEventIgnoredIfNotExpected() throws KeeperException, IOException {
1260
1261 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1262
1263 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1264 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1265 server.getConfiguration());
1266 final AtomicBoolean zkEventProcessed = new AtomicBoolean(false);
1267
1268 AssignmentManager am = new AssignmentManager(this.server,
1269 this.serverManager, ct, balancer, null, null, master.getTableLockManager()) {
1270
1271 @Override
1272 void handleRegion(final RegionTransition rt, int expectedVersion) {
1273 super.handleRegion(rt, expectedVersion);
1274 if (rt != null && Bytes.equals(hri.getRegionName(),
1275 rt.getRegionName()) && rt.getEventType() == EventType.RS_ZK_REGION_OPENING) {
1276 zkEventProcessed.set(true);
1277 }
1278 }
1279 };
1280 try {
1281
1282 am.getRegionStates().regionOffline(hri);
1283 zkEventProcessed.set(false);
1284 this.watcher.registerListenerFirst(am);
1285 assertFalse("The region should not be in transition",
1286 am.getRegionStates().isRegionInTransition(hri));
1287 ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_A);
1288
1289 ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_A);
1290 long startTime = EnvironmentEdgeManager.currentTimeMillis();
1291 while (!zkEventProcessed.get()) {
1292 assertTrue("Timed out in waiting for ZK event to be processed",
1293 EnvironmentEdgeManager.currentTimeMillis() - startTime < 30000);
1294 Threads.sleepWithoutInterrupt(100);
1295 }
1296 assertFalse(am.getRegionStates().isRegionInTransition(hri));
1297 } finally {
1298 am.shutdown();
1299 }
1300 }
1301 }