1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.hbase.CellScannable;
34 import org.apache.hadoop.hbase.CellUtil;
35 import org.apache.hadoop.hbase.HBaseConfiguration;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.MediumTests;
40 import org.apache.hadoop.hbase.RegionException;
41 import org.apache.hadoop.hbase.RegionTransition;
42 import org.apache.hadoop.hbase.Server;
43 import org.apache.hadoop.hbase.ServerLoad;
44 import org.apache.hadoop.hbase.ServerName;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
47 import org.apache.hadoop.hbase.catalog.CatalogTracker;
48 import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
49 import org.apache.hadoop.hbase.client.HConnection;
50 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
51 import org.apache.hadoop.hbase.client.Result;
52 import org.apache.hadoop.hbase.exceptions.DeserializationException;
53 import org.apache.hadoop.hbase.executor.EventType;
54 import org.apache.hadoop.hbase.executor.ExecutorService;
55 import org.apache.hadoop.hbase.executor.ExecutorType;
56 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
57 import org.apache.hadoop.hbase.master.RegionState.State;
58 import org.apache.hadoop.hbase.master.TableLockManager.NullTableLockManager;
59 import org.apache.hadoop.hbase.master.balancer.DefaultLoadBalancer;
60 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
61 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
62 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
63 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
64 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
65 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
66 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
67 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultCellMeta;
68 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
69 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
70 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table;
71 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
72 import org.apache.hadoop.hbase.util.Bytes;
73 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
74 import org.apache.hadoop.hbase.util.Threads;
75 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
76 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
77 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
78 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
79 import org.apache.zookeeper.KeeperException;
80 import org.apache.zookeeper.KeeperException.NodeExistsException;
81 import org.apache.zookeeper.Watcher;
82 import org.junit.After;
83 import org.junit.AfterClass;
84 import org.junit.Before;
85 import org.junit.BeforeClass;
86 import org.junit.Test;
87 import org.junit.experimental.categories.Category;
88 import org.mockito.Mockito;
89 import org.mockito.internal.util.reflection.Whitebox;
90 import org.mockito.invocation.InvocationOnMock;
91 import org.mockito.stubbing.Answer;
92
93 import com.google.protobuf.RpcController;
94 import com.google.protobuf.ServiceException;
95
96
97
98
99
100 @Category(MediumTests.class)
101 public class TestAssignmentManager {
102 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
103 private static final ServerName SERVERNAME_A =
104 new ServerName("example.org", 1234, 5678);
105 private static final ServerName SERVERNAME_B =
106 new ServerName("example.org", 0, 5678);
107 private static final HRegionInfo REGIONINFO =
108 new HRegionInfo(TableName.valueOf("t"),
109 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
110 private static int assignmentCount;
111 private static boolean enabling = false;
112
113
114 private Server server;
115 private ServerManager serverManager;
116 private ZooKeeperWatcher watcher;
117 private LoadBalancer balancer;
118 private HMaster master;
119
120 @BeforeClass
121 public static void beforeClass() throws Exception {
122 HTU.startMiniZKCluster();
123 }
124
125 @AfterClass
126 public static void afterClass() throws IOException {
127 HTU.shutdownMiniZKCluster();
128 }
129
130 @Before
131 public void before() throws ZooKeeperConnectionException, IOException {
132
133
134
135
136
137
138 this.server = Mockito.mock(Server.class);
139 Mockito.when(server.getServerName()).thenReturn(new ServerName("master,1,1"));
140 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
141 this.watcher =
142 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
143 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
144 Mockito.doThrow(new RuntimeException("Aborted")).
145 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
146
147
148
149 this.serverManager = Mockito.mock(ServerManager.class);
150 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
151 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
152 Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer());
153 final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
154 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
155 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
156 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
157 new ArrayList<ServerName>(onlineServers.keySet()));
158 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
159
160 List<ServerName> avServers = new ArrayList<ServerName>();
161 avServers.addAll(onlineServers.keySet());
162 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
163 Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers);
164
165 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
166 thenReturn(true);
167 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
168 thenReturn(true);
169
170 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)).
171 thenReturn(RegionOpeningState.OPENED);
172 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)).
173 thenReturn(RegionOpeningState.OPENED);
174 this.master = Mockito.mock(HMaster.class);
175
176 Mockito.when(this.master.getServerManager()).thenReturn(serverManager);
177 }
178
179 @After
180 public void after() throws KeeperException {
181 if (this.watcher != null) {
182
183 ZKAssign.deleteAllNodes(this.watcher);
184 this.watcher.close();
185 }
186 }
187
188
189
190
191
192
193
194
195
196 @Test(timeout = 60000)
197 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
198 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
199 AssignmentManagerWithExtrasForTesting am =
200 setUpMockedAssignmentManager(this.server, this.serverManager);
201 try {
202 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
203 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
204 while (!am.processRITInvoked) Thread.sleep(1);
205
206
207
208 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
209
210 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
211
212
213
214
215
216
217 int versionid =
218 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
219 assertNotSame(versionid, -1);
220 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
221
222
223
224 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
225 assertNotSame(-1, versionid);
226
227 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
228 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
229 EventType.RS_ZK_REGION_OPENING, versionid);
230 assertNotSame(-1, versionid);
231
232 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
233 SERVERNAME_B, versionid);
234 assertNotSame(-1, versionid);
235 am.gate.set(false);
236
237 ZKAssign.blockUntilNoRIT(watcher);
238 } finally {
239 am.getExecutorService().shutdown();
240 am.shutdown();
241 }
242 }
243
244 @Test(timeout = 60000)
245 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
246 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
247 AssignmentManagerWithExtrasForTesting am =
248 setUpMockedAssignmentManager(this.server, this.serverManager);
249 try {
250 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
251 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
252 while (!am.processRITInvoked) Thread.sleep(1);
253
254
255
256 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
257
258 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
259
260
261
262
263
264
265 int versionid =
266 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
267 assertNotSame(versionid, -1);
268 am.gate.set(false);
269 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
270
271
272
273 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
274 assertNotSame(-1, versionid);
275
276 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
277 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
278 EventType.RS_ZK_REGION_OPENING, versionid);
279 assertNotSame(-1, versionid);
280
281 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
282 SERVERNAME_B, versionid);
283 assertNotSame(-1, versionid);
284
285
286 ZKAssign.blockUntilNoRIT(watcher);
287 } finally {
288 am.getExecutorService().shutdown();
289 am.shutdown();
290 }
291 }
292
293 @Test(timeout = 60000)
294 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
295 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
296 AssignmentManagerWithExtrasForTesting am =
297 setUpMockedAssignmentManager(this.server, this.serverManager);
298 try {
299 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
300 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
301 while (!am.processRITInvoked) Thread.sleep(1);
302
303
304
305 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
306
307 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
308
309
310
311
312
313
314 int versionid =
315 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
316 assertNotSame(versionid, -1);
317 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
318
319 am.gate.set(false);
320
321
322 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
323 assertNotSame(-1, versionid);
324
325 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
326 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
327 EventType.RS_ZK_REGION_OPENING, versionid);
328 assertNotSame(-1, versionid);
329
330 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
331 SERVERNAME_B, versionid);
332 assertNotSame(-1, versionid);
333
334 ZKAssign.blockUntilNoRIT(watcher);
335 } finally {
336 am.getExecutorService().shutdown();
337 am.shutdown();
338 }
339 }
340
341 private void createRegionPlanAndBalance(
342 final AssignmentManager am, final ServerName from,
343 final ServerName to, final HRegionInfo hri) throws RegionException {
344
345
346 am.regionOnline(hri, from);
347
348
349 am.balance(new RegionPlan(hri, from, to));
350 }
351
352
353
354
355
356
357
358
359 @Test
360 public void testBalance()
361 throws IOException, KeeperException, DeserializationException, InterruptedException {
362
363
364 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
365
366
367 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
368 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
369 .getConfiguration());
370
371 AssignmentManager am = new AssignmentManager(this.server,
372 this.serverManager, ct, balancer, executor, null, master.getTableLockManager());
373 am.failoverCleanupDone.set(true);
374 try {
375
376
377 this.watcher.registerListenerFirst(am);
378
379
380 am.regionOnline(REGIONINFO, SERVERNAME_A);
381
382 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
383 am.balance(plan);
384
385 RegionStates regionStates = am.getRegionStates();
386
387 assertTrue(regionStates.isRegionInTransition(REGIONINFO)
388 && regionStates.isRegionInState(REGIONINFO, State.FAILED_CLOSE));
389
390 regionStates.updateRegionState(REGIONINFO, State.PENDING_CLOSE);
391
392
393
394
395
396
397 int versionid =
398 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
399 assertNotSame(versionid, -1);
400
401
402
403
404
405 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
406
407
408 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
409 assertNotSame(-1, versionid);
410
411 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
412 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
413 EventType.RS_ZK_REGION_OPENING, versionid);
414 assertNotSame(-1, versionid);
415
416 versionid =
417 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
418 assertNotSame(-1, versionid);
419
420 while(regionStates.isRegionInTransition(REGIONINFO)) Threads.sleep(1);
421 } finally {
422 executor.shutdown();
423 am.shutdown();
424
425 ZKAssign.deleteAllNodes(this.watcher);
426 }
427 }
428
429
430
431
432
433
434 @Test
435 public void testShutdownHandler()
436 throws KeeperException, IOException, ServiceException {
437
438
439 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
440
441
442 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
443
444 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
445 this.server, this.serverManager);
446 try {
447 processServerShutdownHandler(ct, am, false);
448 } finally {
449 executor.shutdown();
450 am.shutdown();
451
452 ZKAssign.deleteAllNodes(this.watcher);
453 }
454 }
455
456
457
458
459
460
461
462
463
464
465 @Test
466 public void testSSHWhenDisableTableInProgress() throws KeeperException, IOException,
467 ServiceException {
468 testCaseWithPartiallyDisabledState(Table.State.DISABLING);
469 testCaseWithPartiallyDisabledState(Table.State.DISABLED);
470 }
471
472
473
474
475
476
477
478
479
480 @Test
481 public void testSSHWhenSplitRegionInProgress() throws KeeperException, IOException, Exception {
482
483 testCaseWithSplitRegionPartial(true);
484
485 testCaseWithSplitRegionPartial(false);
486 }
487
488 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException,
489 IOException, NodeExistsException, InterruptedException, ServiceException {
490
491
492 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
493
494 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
495 ZKAssign.deleteAllNodes(this.watcher);
496
497
498 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
499 this.server, this.serverManager);
500
501 am.regionOnline(REGIONINFO, SERVERNAME_A);
502
503 am.getRegionStates().updateRegionState(
504 REGIONINFO, State.SPLITTING, SERVERNAME_A);
505 am.getZKTable().setEnabledTable(REGIONINFO.getTableName());
506 RegionTransition data = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
507 REGIONINFO.getRegionName(), SERVERNAME_A);
508 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
509
510 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
511
512 try {
513 processServerShutdownHandler(ct, am, regionSplitDone);
514
515
516
517 if (regionSplitDone) {
518 assertFalse("Region state of region in SPLITTING should be removed from rit.",
519 am.getRegionStates().isRegionsInTransition());
520 } else {
521 while (!am.assignInvoked) {
522 Thread.sleep(1);
523 }
524 assertTrue("Assign should be invoked.", am.assignInvoked);
525 }
526 } finally {
527 REGIONINFO.setOffline(false);
528 REGIONINFO.setSplit(false);
529 executor.shutdown();
530 am.shutdown();
531
532 ZKAssign.deleteAllNodes(this.watcher);
533 }
534 }
535
536 private void testCaseWithPartiallyDisabledState(Table.State state) throws KeeperException,
537 IOException, NodeExistsException, ServiceException {
538
539
540 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
541
542 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
543 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
544 ZKAssign.deleteAllNodes(this.watcher);
545
546
547 AssignmentManager am = new AssignmentManager(this.server,
548 this.serverManager, ct, balancer, executor, null, master.getTableLockManager());
549
550 am.regionOnline(REGIONINFO, SERVERNAME_A);
551
552 am.getRegionStates().updateRegionState(REGIONINFO, State.PENDING_CLOSE);
553 if (state == Table.State.DISABLING) {
554 am.getZKTable().setDisablingTable(REGIONINFO.getTableName());
555 } else {
556 am.getZKTable().setDisabledTable(REGIONINFO.getTableName());
557 }
558 RegionTransition data = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING,
559 REGIONINFO.getRegionName(), SERVERNAME_A);
560
561
562
563 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
564
565 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
566
567 try {
568 processServerShutdownHandler(ct, am, false);
569
570
571 assertTrue("The znode should be deleted.", ZKUtil.checkExists(this.watcher, node) == -1);
572
573
574
575 if (state == Table.State.DISABLED) {
576 assertFalse("Region state of region in pending close should be removed from rit.",
577 am.getRegionStates().isRegionsInTransition());
578 }
579 } finally {
580 am.setEnabledTable(REGIONINFO.getTableName());
581 executor.shutdown();
582 am.shutdown();
583
584 ZKAssign.deleteAllNodes(this.watcher);
585 }
586 }
587
588 private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am, boolean splitRegion)
589 throws IOException, ServiceException {
590
591
592 this.watcher.registerListenerFirst(am);
593
594
595
596 ClientProtos.ClientService.BlockingInterface implementation =
597 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
598
599
600 Result r;
601 if (splitRegion) {
602 r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
603 } else {
604 r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
605 }
606
607 final ScanResponse.Builder builder = ScanResponse.newBuilder();
608 builder.setMoreResults(true);
609 ResultCellMeta.Builder metaBuilder = ResultCellMeta.newBuilder();
610 metaBuilder.addCellsLength(r.size());
611 builder.setResultCellMeta(metaBuilder.build());
612 final List<CellScannable> cellScannables = new ArrayList<CellScannable>(1);
613 cellScannables.add(r);
614 Mockito.when(implementation.scan(
615 (RpcController)Mockito.any(), (ScanRequest)Mockito.any())).
616 thenAnswer(new Answer<ScanResponse>() {
617 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
618 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
619 .getArguments()[0];
620 if (controller != null) {
621 controller.setCellScanner(CellUtil.createCellScanner(cellScannables));
622 }
623 return builder.build();
624 }
625 });
626
627
628 HConnection connection =
629 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
630 null, implementation, SERVERNAME_B, REGIONINFO);
631
632
633
634 Mockito.when(ct.getConnection()).thenReturn(connection);
635 Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
636
637
638
639 DeadServer deadServers = new DeadServer();
640 deadServers.add(SERVERNAME_A);
641
642 MasterServices services = Mockito.mock(MasterServices.class);
643 Mockito.when(services.getAssignmentManager()).thenReturn(am);
644 Mockito.when(services.getServerManager()).thenReturn(this.serverManager);
645 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
646 ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
647 services, deadServers, SERVERNAME_A, false);
648 am.failoverCleanupDone.set(true);
649 handler.process();
650
651 }
652
653
654
655
656
657
658
659 private ExecutorService startupMasterExecutor(final String name) {
660
661 ExecutorService executor = new ExecutorService(name);
662 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
663 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
664 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
665 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
666 return executor;
667 }
668
669 @Test
670 public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
671
672 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
673
674
675
676 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
677
678 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
679 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
680 .getConfiguration());
681
682 AssignmentManager am = new AssignmentManager(this.server,
683 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
684 try {
685
686 unassign(am, SERVERNAME_A, hri);
687
688 ZKAssign.deleteClosingNode(this.watcher, hri);
689
690
691
692 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
693
694
695
696
697 unassign(am, SERVERNAME_A, hri);
698
699 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
700 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
701 assertFalse(am.getRegionStates().isRegionInTransition(hri));
702 } finally {
703 am.shutdown();
704 }
705 }
706
707
708
709
710
711
712
713 @Test(timeout = 60000)
714 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
715 throws IOException, KeeperException, InterruptedException, ServiceException {
716 final RecoverableZooKeeper recoverableZk = Mockito
717 .mock(RecoverableZooKeeper.class);
718 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
719 this.server, this.serverManager);
720 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
721 null) {
722 public RecoverableZooKeeper getRecoverableZooKeeper() {
723 return recoverableZk;
724 }
725 };
726 ((ZooKeeperWatcher) zkw).registerListener(am);
727 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
728 .getChildren("/hbase/region-in-transition", null);
729 am.setWatcher((ZooKeeperWatcher) zkw);
730 try {
731 am.processDeadServersAndRegionsInTransition(null);
732 fail("Expected to abort");
733 } catch (NullPointerException e) {
734 fail("Should not throw NPE");
735 } catch (RuntimeException e) {
736 assertEquals("Aborted", e.getLocalizedMessage());
737 }
738 }
739
740
741
742
743 @Test(timeout = 60000)
744 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
745 ServiceException, InterruptedException {
746 this.server.getConfiguration().setClass(
747 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockedLoadBalancer.class,
748 LoadBalancer.class);
749 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
750 this.server, this.serverManager);
751 try {
752
753
754
755 AtomicBoolean gate = new AtomicBoolean(false);
756 if (balancer instanceof MockedLoadBalancer) {
757 ((MockedLoadBalancer) balancer).setGateVariable(gate);
758 }
759 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
760 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
761 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A,
762 EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
763 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO
764 .getEncodedName());
765 am.getRegionStates().updateRegionState(
766 REGIONINFO, State.OPENING, SERVERNAME_A);
767
768
769 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(
770 REGIONINFO, null, SERVERNAME_A));
771 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
772 List<ServerName> serverList = new ArrayList<ServerName>(2);
773 serverList.add(SERVERNAME_B);
774 Mockito.when(
775 this.serverManager.createDestinationServersList(SERVERNAME_A))
776 .thenReturn(serverList);
777 am.nodeDataChanged(path);
778
779
780 while (!gate.get()) {
781 Thread.sleep(10);
782 }
783
784
785
786 RegionPlan newRegionPlan = am.regionPlans
787 .get(REGIONINFO.getEncodedName());
788 while (newRegionPlan == null) {
789 Thread.sleep(10);
790 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
791 }
792
793
794
795 assertNotSame("Same region plan should not come", regionPlan,
796 newRegionPlan);
797 assertTrue("Destination servers should be different.", !(regionPlan
798 .getDestination().equals(newRegionPlan.getDestination())));
799
800 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
801 } finally {
802 this.server.getConfiguration().setClass(
803 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, DefaultLoadBalancer.class,
804 LoadBalancer.class);
805 am.getExecutorService().shutdown();
806 am.shutdown();
807 }
808 }
809
810
811
812
813
814 public static class MockedLoadBalancer extends DefaultLoadBalancer {
815 private AtomicBoolean gate;
816
817 public void setGateVariable(AtomicBoolean gate) {
818 this.gate = gate;
819 }
820
821 @Override
822 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
823 ServerName randomServerName = super.randomAssignment(regionInfo, servers);
824 this.gate.set(true);
825 return randomServerName;
826 }
827
828 @Override
829 public Map<ServerName, List<HRegionInfo>> retainAssignment(
830 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
831 this.gate.set(true);
832 return super.retainAssignment(regions, servers);
833 }
834 }
835
836
837
838
839
840
841 @Test(timeout = 60000)
842 public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException,
843 KeeperException, ServiceException, InterruptedException {
844 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
845 this.server, this.serverManager);
846 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
847 int version = ZKAssign.getVersion(this.watcher, REGIONINFO);
848 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
849 EventType.RS_ZK_REGION_OPENING, version);
850 RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING,
851 REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY);
852 version = ZKAssign.getVersion(this.watcher, REGIONINFO);
853 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
854 am.getRegionStates().createRegionState(REGIONINFO);
855 am.gate.set(false);
856 am.processRegionsInTransition(rt, REGIONINFO, version);
857
858 while (!am.gate.get()) {
859 Thread.sleep(10);
860 }
861 assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO
862 .getEncodedName()));
863 }
864
865
866
867
868
869
870
871
872
873 @Test(timeout = 60000)
874 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
875 throws KeeperException, IOException, Exception {
876 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
877 MockedLoadBalancer.class, LoadBalancer.class);
878 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
879 new HashMap<ServerName, ServerLoad>(0));
880 List<ServerName> destServers = new ArrayList<ServerName>(1);
881 destServers.add(SERVERNAME_A);
882 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
883
884 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
885 Server server = new HMaster(HTU.getConfiguration());
886 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
887 this.serverManager);
888 AtomicBoolean gate = new AtomicBoolean(false);
889 if (balancer instanceof MockedLoadBalancer) {
890 ((MockedLoadBalancer) balancer).setGateVariable(gate);
891 }
892 try{
893
894 am.getZKTable().setDisablingTable(REGIONINFO.getTableName());
895 am.joinCluster();
896
897 assertFalse(
898 "Assign should not be invoked for disabling table regions during clean cluster startup.",
899 gate.get());
900
901 assertTrue("Table should be disabled.",
902 am.getZKTable().isDisabledTable(REGIONINFO.getTableName()));
903 } finally {
904 this.server.getConfiguration().setClass(
905 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, DefaultLoadBalancer.class,
906 LoadBalancer.class);
907 am.getZKTable().setEnabledTable(REGIONINFO.getTableName());
908 am.shutdown();
909 }
910 }
911
912
913
914
915
916
917
918
919 @Test
920 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
921 enabling = true;
922 List<ServerName> destServers = new ArrayList<ServerName>(1);
923 destServers.add(SERVERNAME_A);
924 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
925 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
926 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
927 Server server = new HMaster(HTU.getConfiguration());
928 Whitebox.setInternalState(server, "serverManager", this.serverManager);
929 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
930 this.serverManager);
931 try {
932
933 am.getZKTable().setEnablingTable(REGIONINFO.getTableName());
934 new EnableTableHandler(server, REGIONINFO.getTableName(),
935 am.getCatalogTracker(), am, new NullTableLockManager(), true).prepare()
936 .process();
937 assertEquals("Number of assignments should be 1.", 1, assignmentCount);
938 assertTrue("Table should be enabled.",
939 am.getZKTable().isEnabledTable(REGIONINFO.getTableName()));
940 } finally {
941 enabling = false;
942 assignmentCount = 0;
943 am.getZKTable().setEnabledTable(REGIONINFO.getTableName());
944 am.shutdown();
945 ZKAssign.deleteAllNodes(this.watcher);
946 }
947 }
948
949
950
951
952
953
954 @Test
955 public void testSSHTimesOutOpeningRegionTransition()
956 throws KeeperException, IOException, ServiceException {
957
958 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
959
960 AssignmentManagerWithExtrasForTesting am =
961 setUpMockedAssignmentManager(this.server, this.serverManager);
962
963 RegionState state = new RegionState(REGIONINFO,
964 State.OPENING, System.currentTimeMillis(), SERVERNAME_A);
965 am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_B);
966 am.getRegionStates().regionsInTransition.put(REGIONINFO.getEncodedName(), state);
967
968 am.regionPlans.put(REGIONINFO.getEncodedName(),
969 new RegionPlan(REGIONINFO, SERVERNAME_B, SERVERNAME_A));
970 am.getZKTable().setEnabledTable(REGIONINFO.getTableName());
971
972 try {
973 am.assignInvoked = false;
974 processServerShutdownHandler(ct, am, false);
975 assertTrue(am.assignInvoked);
976 } finally {
977 am.getRegionStates().regionsInTransition.remove(REGIONINFO.getEncodedName());
978 am.regionPlans.remove(REGIONINFO.getEncodedName());
979 }
980 }
981
982
983
984
985
986
987
988
989
990
991 @Test
992 public void testClosingFailureDuringRecovery() throws Exception {
993
994 AssignmentManagerWithExtrasForTesting am =
995 setUpMockedAssignmentManager(this.server, this.serverManager);
996 ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_A);
997 am.getRegionStates().createRegionState(REGIONINFO);
998
999 assertFalse( am.getRegionStates().isRegionsInTransition() );
1000
1001 am.processRegionInTransition(REGIONINFO.getEncodedName(), REGIONINFO);
1002
1003 assertTrue( am.getRegionStates().isRegionsInTransition() );
1004 }
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
1023 final HRegionInfo region, final ServerName serverName)
1024 throws KeeperException, IOException {
1025 RegionTransition rt =
1026 RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
1027 region.getRegionName(), serverName);
1028
1029 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1030 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1031 throw new IOException("Failed create of ephemeral " + node);
1032 }
1033
1034
1035 return transitionNodeSplitting(zkw, region, serverName, -1);
1036 }
1037
1038
1039
1040 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
1041 final HRegionInfo parent,
1042 final ServerName serverName, final int version)
1043 throws KeeperException, IOException {
1044 return ZKAssign.transitionNode(zkw, parent, serverName,
1045 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
1046 }
1047
1048 private void unassign(final AssignmentManager am, final ServerName sn,
1049 final HRegionInfo hri) throws RegionException {
1050
1051 am.regionOnline(hri, sn);
1052
1053 am.unassign(hri);
1054 }
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
1066 final ServerManager manager) throws IOException, KeeperException, ServiceException {
1067
1068 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1069
1070
1071
1072
1073
1074 ClientProtos.ClientService.BlockingInterface ri =
1075 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
1076
1077 Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
1078 final ScanResponse.Builder builder = ScanResponse.newBuilder();
1079 builder.setMoreResults(true);
1080 ResultCellMeta.Builder metaBuilder = ResultCellMeta.newBuilder();
1081 metaBuilder.addCellsLength(r.size());
1082 builder.setResultCellMeta(metaBuilder.build());
1083 final List<CellScannable> rows = new ArrayList<CellScannable>(1);
1084 rows.add(r);
1085 Answer<ScanResponse> ans = new Answer<ClientProtos.ScanResponse>() {
1086 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
1087 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
1088 .getArguments()[0];
1089 if (controller != null) {
1090 controller.setCellScanner(CellUtil.createCellScanner(rows));
1091 }
1092 return builder.build();
1093 }
1094 };
1095 if (enabling) {
1096 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any()))
1097 .thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans)
1098 .thenReturn(ScanResponse.newBuilder().setMoreResults(false).build());
1099 } else {
1100 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any())).thenAnswer(
1101 ans);
1102 }
1103
1104 GetResponse.Builder getBuilder = GetResponse.newBuilder();
1105 getBuilder.setResult(ProtobufUtil.toResult(r));
1106 Mockito.when(ri.get((RpcController)Mockito.any(), (GetRequest) Mockito.any())).
1107 thenReturn(getBuilder.build());
1108
1109 HConnection connection = HConnectionTestingUtility.
1110 getMockedConnectionAndDecorate(HTU.getConfiguration(), null,
1111 ri, SERVERNAME_B, REGIONINFO);
1112
1113 Mockito.when(ct.getConnection()).thenReturn(connection);
1114
1115 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
1116 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
1117 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
1118 server, manager, ct, this.balancer, executor, new NullTableLockManager());
1119 return am;
1120 }
1121
1122
1123
1124
1125 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1126
1127 private final ExecutorService es;
1128
1129 private final CatalogTracker ct;
1130 boolean processRITInvoked = false;
1131 boolean assignInvoked = false;
1132 AtomicBoolean gate = new AtomicBoolean(true);
1133
1134 public AssignmentManagerWithExtrasForTesting(
1135 final Server master, final ServerManager serverManager,
1136 final CatalogTracker catalogTracker, final LoadBalancer balancer,
1137 final ExecutorService service, final TableLockManager tableLockManager)
1138 throws KeeperException, IOException {
1139 super(master, serverManager, catalogTracker, balancer, service, null, tableLockManager);
1140 this.es = service;
1141 this.ct = catalogTracker;
1142 }
1143
1144 @Override
1145 boolean processRegionInTransition(String encodedRegionName,
1146 HRegionInfo regionInfo) throws KeeperException, IOException {
1147 this.processRITInvoked = true;
1148 return super.processRegionInTransition(encodedRegionName, regionInfo);
1149 }
1150
1151 @Override
1152 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan) {
1153 if (enabling) {
1154 assignmentCount++;
1155 this.regionOnline(region, SERVERNAME_A);
1156 } else {
1157 super.assign(region, setOfflineInZK, forceNewPlan);
1158 this.gate.set(true);
1159 }
1160 }
1161
1162 @Override
1163 public void assign(List<HRegionInfo> regions)
1164 throws IOException, InterruptedException {
1165 assignInvoked = (regions != null && regions.size() > 0);
1166 }
1167
1168
1169 void setWatcher(ZooKeeperWatcher watcher) {
1170 this.watcher = watcher;
1171 }
1172
1173
1174
1175
1176 ExecutorService getExecutorService() {
1177 return this.es;
1178 }
1179
1180
1181
1182
1183 CatalogTracker getCatalogTracker() {
1184 return this.ct;
1185 }
1186 }
1187
1188
1189
1190
1191
1192
1193
1194 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1195 final ZooKeeperWatcher watcher) {
1196
1197
1198 watcher.registerListenerFirst(am);
1199 Thread t = new Thread("RunAmJoinCluster") {
1200 public void run() {
1201
1202
1203
1204
1205
1206 am.getRegionStates().regionsInTransition.clear();
1207 am.regionPlans.clear();
1208 try {
1209 am.joinCluster();
1210 } catch (IOException e) {
1211 throw new RuntimeException(e);
1212 } catch (KeeperException e) {
1213 throw new RuntimeException(e);
1214 } catch (InterruptedException e) {
1215 throw new RuntimeException(e);
1216 }
1217 }
1218 };
1219 t.start();
1220 while (!t.isAlive()) Threads.sleep(1);
1221 }
1222
1223 @Test
1224 public void testForceAssignMergingRegion() throws Exception {
1225
1226 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1227
1228 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1229 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1230 server.getConfiguration());
1231
1232 AssignmentManager am = new AssignmentManager(this.server,
1233 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
1234 RegionStates regionStates = am.getRegionStates();
1235 try {
1236
1237 regionStates.updateRegionState(hri, RegionState.State.MERGING);
1238
1239 am.assign(hri, true, true);
1240 assertEquals("The region should be still in merging state",
1241 RegionState.State.MERGING, regionStates.getRegionState(hri).getState());
1242 } finally {
1243 am.shutdown();
1244 }
1245 }
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257 @Test
1258 public void testAssignmentEventIgnoredIfNotExpected() throws KeeperException, IOException {
1259
1260 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1261
1262 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1263 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1264 server.getConfiguration());
1265 final AtomicBoolean zkEventProcessed = new AtomicBoolean(false);
1266
1267 AssignmentManager am = new AssignmentManager(this.server,
1268 this.serverManager, ct, balancer, null, null, master.getTableLockManager()) {
1269
1270 @Override
1271 void handleRegion(final RegionTransition rt, int expectedVersion) {
1272 super.handleRegion(rt, expectedVersion);
1273 if (rt != null && Bytes.equals(hri.getRegionName(),
1274 rt.getRegionName()) && rt.getEventType() == EventType.RS_ZK_REGION_OPENING) {
1275 zkEventProcessed.set(true);
1276 }
1277 }
1278 };
1279 try {
1280
1281 am.getRegionStates().regionOffline(hri);
1282 zkEventProcessed.set(false);
1283 this.watcher.registerListenerFirst(am);
1284 assertFalse("The region should not be in transition",
1285 am.getRegionStates().isRegionInTransition(hri));
1286 ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_A);
1287
1288 ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_A);
1289 long startTime = EnvironmentEdgeManager.currentTimeMillis();
1290 while (!zkEventProcessed.get()) {
1291 assertTrue("Timed out in waiting for ZK event to be processed",
1292 EnvironmentEdgeManager.currentTimeMillis() - startTime < 30000);
1293 Threads.sleepWithoutInterrupt(100);
1294 }
1295 assertFalse(am.getRegionStates().isRegionInTransition(hri));
1296 } finally {
1297 am.shutdown();
1298 }
1299 }
1300 }