1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.hbase.CellScannable;
34 import org.apache.hadoop.hbase.CellUtil;
35 import org.apache.hadoop.hbase.HBaseConfiguration;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.MediumTests;
40 import org.apache.hadoop.hbase.RegionException;
41 import org.apache.hadoop.hbase.RegionTransition;
42 import org.apache.hadoop.hbase.Server;
43 import org.apache.hadoop.hbase.ServerLoad;
44 import org.apache.hadoop.hbase.ServerName;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
47 import org.apache.hadoop.hbase.catalog.CatalogTracker;
48 import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
49 import org.apache.hadoop.hbase.client.HConnection;
50 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
51 import org.apache.hadoop.hbase.client.Result;
52 import org.apache.hadoop.hbase.exceptions.DeserializationException;
53 import org.apache.hadoop.hbase.executor.EventType;
54 import org.apache.hadoop.hbase.executor.ExecutorService;
55 import org.apache.hadoop.hbase.executor.ExecutorType;
56 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
57 import org.apache.hadoop.hbase.master.RegionState.State;
58 import org.apache.hadoop.hbase.master.TableLockManager.NullTableLockManager;
59 import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
60 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
61 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
62 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
63 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
64 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
65 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
66 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
67 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
68 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
69 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table;
70 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
71 import org.apache.hadoop.hbase.util.Bytes;
72 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73 import org.apache.hadoop.hbase.util.Threads;
74 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
75 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
76 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
77 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
78 import org.apache.zookeeper.KeeperException;
79 import org.apache.zookeeper.KeeperException.NodeExistsException;
80 import org.apache.zookeeper.Watcher;
81 import org.junit.After;
82 import org.junit.AfterClass;
83 import org.junit.Before;
84 import org.junit.BeforeClass;
85 import org.junit.Test;
86 import org.junit.experimental.categories.Category;
87 import org.mockito.Mockito;
88 import org.mockito.internal.util.reflection.Whitebox;
89 import org.mockito.invocation.InvocationOnMock;
90 import org.mockito.stubbing.Answer;
91
92 import com.google.protobuf.RpcController;
93 import com.google.protobuf.ServiceException;
94
95
96
97
98
99 @Category(MediumTests.class)
100 public class TestAssignmentManager {
101 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
102 private static final ServerName SERVERNAME_A =
103 ServerName.valueOf("example.org", 1234, 5678);
104 private static final ServerName SERVERNAME_B =
105 ServerName.valueOf("example.org", 0, 5678);
106 private static final HRegionInfo REGIONINFO =
107 new HRegionInfo(TableName.valueOf("t"),
108 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
109 private static int assignmentCount;
110 private static boolean enabling = false;
111
112
113 private Server server;
114 private ServerManager serverManager;
115 private ZooKeeperWatcher watcher;
116 private LoadBalancer balancer;
117 private HMaster master;
118
119 @BeforeClass
120 public static void beforeClass() throws Exception {
121 HTU.startMiniZKCluster();
122 }
123
124 @AfterClass
125 public static void afterClass() throws IOException {
126 HTU.shutdownMiniZKCluster();
127 }
128
129 @Before
130 public void before() throws ZooKeeperConnectionException, IOException {
131
132
133
134
135
136
137 this.server = Mockito.mock(Server.class);
138 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
139 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
140 this.watcher =
141 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
142 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
143 Mockito.doThrow(new RuntimeException("Aborted")).
144 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
145
146
147
148 this.serverManager = Mockito.mock(ServerManager.class);
149 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
150 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
151 Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer());
152 final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
153 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
154 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
155 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
156 new ArrayList<ServerName>(onlineServers.keySet()));
157 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
158
159 List<ServerName> avServers = new ArrayList<ServerName>();
160 avServers.addAll(onlineServers.keySet());
161 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
162 Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers);
163
164 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
165 thenReturn(true);
166 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
167 thenReturn(true);
168
169 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)).
170 thenReturn(RegionOpeningState.OPENED);
171 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)).
172 thenReturn(RegionOpeningState.OPENED);
173 this.master = Mockito.mock(HMaster.class);
174
175 Mockito.when(this.master.getServerManager()).thenReturn(serverManager);
176 }
177
178 @After
179 public void after() throws KeeperException {
180 if (this.watcher != null) {
181
182 ZKAssign.deleteAllNodes(this.watcher);
183 this.watcher.close();
184 }
185 }
186
187
188
189
190
191
192
193
194
195 @Test(timeout = 60000)
196 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
197 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
198 AssignmentManagerWithExtrasForTesting am =
199 setUpMockedAssignmentManager(this.server, this.serverManager);
200 try {
201 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
202 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
203 while (!am.processRITInvoked) Thread.sleep(1);
204
205
206
207 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
208
209 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
210
211
212
213
214
215
216 int versionid =
217 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
218 assertNotSame(versionid, -1);
219 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
220
221
222
223 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
224 assertNotSame(-1, versionid);
225
226 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
227 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
228 EventType.RS_ZK_REGION_OPENING, versionid);
229 assertNotSame(-1, versionid);
230
231 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
232 SERVERNAME_B, versionid);
233 assertNotSame(-1, versionid);
234 am.gate.set(false);
235
236 ZKAssign.blockUntilNoRIT(watcher);
237 } finally {
238 am.getExecutorService().shutdown();
239 am.shutdown();
240 }
241 }
242
243 @Test(timeout = 60000)
244 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
245 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
246 AssignmentManagerWithExtrasForTesting am =
247 setUpMockedAssignmentManager(this.server, this.serverManager);
248 try {
249 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
250 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
251 while (!am.processRITInvoked) Thread.sleep(1);
252
253
254
255 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
256
257 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
258
259
260
261
262
263
264 int versionid =
265 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
266 assertNotSame(versionid, -1);
267 am.gate.set(false);
268 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
269
270
271
272 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
273 assertNotSame(-1, versionid);
274
275 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
276 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
277 EventType.RS_ZK_REGION_OPENING, versionid);
278 assertNotSame(-1, versionid);
279
280 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
281 SERVERNAME_B, versionid);
282 assertNotSame(-1, versionid);
283
284
285 ZKAssign.blockUntilNoRIT(watcher);
286 } finally {
287 am.getExecutorService().shutdown();
288 am.shutdown();
289 }
290 }
291
292 @Test(timeout = 60000)
293 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
294 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
295 AssignmentManagerWithExtrasForTesting am =
296 setUpMockedAssignmentManager(this.server, this.serverManager);
297 try {
298 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
299 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
300 while (!am.processRITInvoked) Thread.sleep(1);
301
302
303
304 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
305
306 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
307
308
309
310
311
312
313 int versionid =
314 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
315 assertNotSame(versionid, -1);
316 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
317
318 am.gate.set(false);
319
320
321 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
322 assertNotSame(-1, versionid);
323
324 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
325 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
326 EventType.RS_ZK_REGION_OPENING, versionid);
327 assertNotSame(-1, versionid);
328
329 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
330 SERVERNAME_B, versionid);
331 assertNotSame(-1, versionid);
332
333 ZKAssign.blockUntilNoRIT(watcher);
334 } finally {
335 am.getExecutorService().shutdown();
336 am.shutdown();
337 }
338 }
339
340 private void createRegionPlanAndBalance(
341 final AssignmentManager am, final ServerName from,
342 final ServerName to, final HRegionInfo hri) throws RegionException {
343
344
345 am.regionOnline(hri, from);
346
347
348 am.balance(new RegionPlan(hri, from, to));
349 }
350
351
352
353
354
355
356
357
358 @Test
359 public void testBalance()
360 throws IOException, KeeperException, DeserializationException, InterruptedException {
361
362
363 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
364
365
366 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
367 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
368 .getConfiguration());
369
370 AssignmentManager am = new AssignmentManager(this.server,
371 this.serverManager, ct, balancer, executor, null, master.getTableLockManager());
372 am.failoverCleanupDone.set(true);
373 try {
374
375
376 this.watcher.registerListenerFirst(am);
377
378
379 am.regionOnline(REGIONINFO, SERVERNAME_A);
380
381 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
382 am.balance(plan);
383
384 RegionStates regionStates = am.getRegionStates();
385
386 assertTrue(regionStates.isRegionInTransition(REGIONINFO)
387 && regionStates.isRegionInState(REGIONINFO, State.FAILED_CLOSE));
388
389 regionStates.updateRegionState(REGIONINFO, State.PENDING_CLOSE);
390
391
392
393
394
395
396 int versionid =
397 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
398 assertNotSame(versionid, -1);
399
400
401
402
403
404 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
405
406
407 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
408 assertNotSame(-1, versionid);
409
410 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
411 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
412 EventType.RS_ZK_REGION_OPENING, versionid);
413 assertNotSame(-1, versionid);
414
415 versionid =
416 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
417 assertNotSame(-1, versionid);
418
419 while(regionStates.isRegionInTransition(REGIONINFO)) Threads.sleep(1);
420 } finally {
421 executor.shutdown();
422 am.shutdown();
423
424 ZKAssign.deleteAllNodes(this.watcher);
425 }
426 }
427
428
429
430
431
432
433 @Test
434 public void testShutdownHandler()
435 throws KeeperException, IOException, ServiceException {
436
437
438 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
439
440
441 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
442
443 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
444 this.server, this.serverManager);
445 try {
446 processServerShutdownHandler(ct, am, false);
447 } finally {
448 executor.shutdown();
449 am.shutdown();
450
451 ZKAssign.deleteAllNodes(this.watcher);
452 }
453 }
454
455
456
457
458
459
460
461
462
463
464 @Test
465 public void testSSHWhenDisableTableInProgress() throws KeeperException, IOException,
466 ServiceException {
467 testCaseWithPartiallyDisabledState(Table.State.DISABLING);
468 testCaseWithPartiallyDisabledState(Table.State.DISABLED);
469 }
470
471
472
473
474
475
476
477
478
479 @Test
480 public void testSSHWhenSplitRegionInProgress() throws KeeperException, IOException, Exception {
481
482 testCaseWithSplitRegionPartial(true);
483
484 testCaseWithSplitRegionPartial(false);
485 }
486
487 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException,
488 IOException, NodeExistsException, InterruptedException, ServiceException {
489
490
491 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
492
493 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
494 ZKAssign.deleteAllNodes(this.watcher);
495
496
497 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
498 this.server, this.serverManager);
499
500 am.regionOnline(REGIONINFO, SERVERNAME_A);
501
502 am.getRegionStates().updateRegionState(
503 REGIONINFO, State.SPLITTING, SERVERNAME_A);
504 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
505 RegionTransition data = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
506 REGIONINFO.getRegionName(), SERVERNAME_A);
507 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
508
509 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
510
511 try {
512 processServerShutdownHandler(ct, am, regionSplitDone);
513
514
515
516 if (regionSplitDone) {
517 assertFalse("Region state of region in SPLITTING should be removed from rit.",
518 am.getRegionStates().isRegionsInTransition());
519 } else {
520 while (!am.assignInvoked) {
521 Thread.sleep(1);
522 }
523 assertTrue("Assign should be invoked.", am.assignInvoked);
524 }
525 } finally {
526 REGIONINFO.setOffline(false);
527 REGIONINFO.setSplit(false);
528 executor.shutdown();
529 am.shutdown();
530
531 ZKAssign.deleteAllNodes(this.watcher);
532 }
533 }
534
535 private void testCaseWithPartiallyDisabledState(Table.State state) throws KeeperException,
536 IOException, NodeExistsException, ServiceException {
537
538
539 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
540
541 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
542 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
543 ZKAssign.deleteAllNodes(this.watcher);
544
545
546 AssignmentManager am = new AssignmentManager(this.server,
547 this.serverManager, ct, balancer, executor, null, master.getTableLockManager());
548
549 am.regionOnline(REGIONINFO, SERVERNAME_A);
550
551 am.getRegionStates().updateRegionState(REGIONINFO, State.PENDING_CLOSE);
552 if (state == Table.State.DISABLING) {
553 am.getZKTable().setDisablingTable(REGIONINFO.getTable());
554 } else {
555 am.getZKTable().setDisabledTable(REGIONINFO.getTable());
556 }
557 RegionTransition data = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING,
558 REGIONINFO.getRegionName(), SERVERNAME_A);
559
560
561
562 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
563
564 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
565
566 try {
567 processServerShutdownHandler(ct, am, false);
568
569
570 assertTrue("The znode should be deleted.", ZKUtil.checkExists(this.watcher, node) == -1);
571
572
573
574 if (state == Table.State.DISABLED) {
575 assertFalse("Region state of region in pending close should be removed from rit.",
576 am.getRegionStates().isRegionsInTransition());
577 }
578 } finally {
579 am.setEnabledTable(REGIONINFO.getTable());
580 executor.shutdown();
581 am.shutdown();
582
583 ZKAssign.deleteAllNodes(this.watcher);
584 }
585 }
586
587 private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am, boolean splitRegion)
588 throws IOException, ServiceException {
589
590
591 this.watcher.registerListenerFirst(am);
592
593
594
595 ClientProtos.ClientService.BlockingInterface implementation =
596 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
597
598
599 Result r;
600 if (splitRegion) {
601 r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
602 } else {
603 r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
604 }
605
606 final ScanResponse.Builder builder = ScanResponse.newBuilder();
607 builder.setMoreResults(true);
608 builder.addCellsPerResult(r.size());
609 final List<CellScannable> cellScannables = new ArrayList<CellScannable>(1);
610 cellScannables.add(r);
611 Mockito.when(implementation.scan(
612 (RpcController)Mockito.any(), (ScanRequest)Mockito.any())).
613 thenAnswer(new Answer<ScanResponse>() {
614 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
615 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
616 .getArguments()[0];
617 if (controller != null) {
618 controller.setCellScanner(CellUtil.createCellScanner(cellScannables));
619 }
620 return builder.build();
621 }
622 });
623
624
625 HConnection connection =
626 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
627 null, implementation, SERVERNAME_B, REGIONINFO);
628
629
630
631 Mockito.when(ct.getConnection()).thenReturn(connection);
632 Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
633
634
635
636 DeadServer deadServers = new DeadServer();
637 deadServers.add(SERVERNAME_A);
638
639 MasterServices services = Mockito.mock(MasterServices.class);
640 Mockito.when(services.getAssignmentManager()).thenReturn(am);
641 Mockito.when(services.getServerManager()).thenReturn(this.serverManager);
642 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
643 ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
644 services, deadServers, SERVERNAME_A, false);
645 am.failoverCleanupDone.set(true);
646 handler.process();
647
648 }
649
650
651
652
653
654
655
656 private ExecutorService startupMasterExecutor(final String name) {
657
658 ExecutorService executor = new ExecutorService(name);
659 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
660 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
661 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
662 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
663 return executor;
664 }
665
666 @Test
667 public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
668
669 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
670
671
672
673 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
674
675 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
676 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
677 .getConfiguration());
678
679 AssignmentManager am = new AssignmentManager(this.server,
680 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
681 try {
682
683 unassign(am, SERVERNAME_A, hri);
684
685 ZKAssign.deleteClosingNode(this.watcher, hri, SERVERNAME_A);
686
687
688
689 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
690
691
692
693
694 unassign(am, SERVERNAME_A, hri);
695
696 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
697 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
698 assertFalse(am.getRegionStates().isRegionInTransition(hri));
699 } finally {
700 am.shutdown();
701 }
702 }
703
704
705
706
707
708
709
710 @Test(timeout = 60000)
711 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
712 throws IOException, KeeperException, InterruptedException, ServiceException {
713 final RecoverableZooKeeper recoverableZk = Mockito
714 .mock(RecoverableZooKeeper.class);
715 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
716 this.server, this.serverManager);
717 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
718 null) {
719 public RecoverableZooKeeper getRecoverableZooKeeper() {
720 return recoverableZk;
721 }
722 };
723 ((ZooKeeperWatcher) zkw).registerListener(am);
724 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
725 .getChildren("/hbase/region-in-transition", null);
726 am.setWatcher((ZooKeeperWatcher) zkw);
727 try {
728 am.processDeadServersAndRegionsInTransition(null);
729 fail("Expected to abort");
730 } catch (NullPointerException e) {
731 fail("Should not throw NPE");
732 } catch (RuntimeException e) {
733 assertEquals("Aborted", e.getLocalizedMessage());
734 }
735 }
736
737
738
739
740 @Test(timeout = 60000)
741 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
742 ServiceException, InterruptedException {
743 this.server.getConfiguration().setClass(
744 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockedLoadBalancer.class,
745 LoadBalancer.class);
746 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
747 this.server, this.serverManager);
748 try {
749
750
751
752 AtomicBoolean gate = new AtomicBoolean(false);
753 if (balancer instanceof MockedLoadBalancer) {
754 ((MockedLoadBalancer) balancer).setGateVariable(gate);
755 }
756 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
757 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
758 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A,
759 EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
760 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO
761 .getEncodedName());
762 am.getRegionStates().updateRegionState(
763 REGIONINFO, State.OPENING, SERVERNAME_A);
764
765
766 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(
767 REGIONINFO, null, SERVERNAME_A));
768 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
769 List<ServerName> serverList = new ArrayList<ServerName>(2);
770 serverList.add(SERVERNAME_B);
771 Mockito.when(
772 this.serverManager.createDestinationServersList(SERVERNAME_A))
773 .thenReturn(serverList);
774 am.nodeDataChanged(path);
775
776
777 while (!gate.get()) {
778 Thread.sleep(10);
779 }
780
781
782
783 RegionPlan newRegionPlan = am.regionPlans
784 .get(REGIONINFO.getEncodedName());
785 while (newRegionPlan == null) {
786 Thread.sleep(10);
787 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
788 }
789
790
791
792 assertNotSame("Same region plan should not come", regionPlan,
793 newRegionPlan);
794 assertTrue("Destination servers should be different.", !(regionPlan
795 .getDestination().equals(newRegionPlan.getDestination())));
796
797 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
798 } finally {
799 this.server.getConfiguration().setClass(
800 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SimpleLoadBalancer.class,
801 LoadBalancer.class);
802 am.getExecutorService().shutdown();
803 am.shutdown();
804 }
805 }
806
807
808
809
810
811 public static class MockedLoadBalancer extends SimpleLoadBalancer {
812 private AtomicBoolean gate;
813
814 public void setGateVariable(AtomicBoolean gate) {
815 this.gate = gate;
816 }
817
818 @Override
819 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
820 ServerName randomServerName = super.randomAssignment(regionInfo, servers);
821 this.gate.set(true);
822 return randomServerName;
823 }
824
825 @Override
826 public Map<ServerName, List<HRegionInfo>> retainAssignment(
827 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
828 this.gate.set(true);
829 return super.retainAssignment(regions, servers);
830 }
831 }
832
833
834
835
836
837
838 @Test(timeout = 60000)
839 public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException,
840 KeeperException, ServiceException, InterruptedException {
841 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
842 this.server, this.serverManager);
843 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
844 int version = ZKAssign.getVersion(this.watcher, REGIONINFO);
845 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
846 EventType.RS_ZK_REGION_OPENING, version);
847 RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING,
848 REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY);
849 version = ZKAssign.getVersion(this.watcher, REGIONINFO);
850 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
851 am.getRegionStates().logSplit(SERVERNAME_A);
852 am.getRegionStates().createRegionState(REGIONINFO);
853 am.gate.set(false);
854 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
855 assertFalse(am.processRegionsInTransition(rt, REGIONINFO, version));
856 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
857 processServerShutdownHandler(ct, am, false);
858
859 while (!am.gate.get()) {
860 Thread.sleep(10);
861 }
862 assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO
863 .getEncodedName()));
864 }
865
866
867
868
869
870
871
872
873
874 @Test(timeout = 60000)
875 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
876 throws KeeperException, IOException, Exception {
877 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
878 MockedLoadBalancer.class, LoadBalancer.class);
879 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
880 new HashMap<ServerName, ServerLoad>(0));
881 List<ServerName> destServers = new ArrayList<ServerName>(1);
882 destServers.add(SERVERNAME_A);
883 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
884
885 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
886 Server server = new HMaster(HTU.getConfiguration());
887 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
888 this.serverManager);
889 AtomicBoolean gate = new AtomicBoolean(false);
890 if (balancer instanceof MockedLoadBalancer) {
891 ((MockedLoadBalancer) balancer).setGateVariable(gate);
892 }
893 try{
894
895 am.getZKTable().setDisablingTable(REGIONINFO.getTable());
896 am.joinCluster();
897
898 assertFalse(
899 "Assign should not be invoked for disabling table regions during clean cluster startup.",
900 gate.get());
901
902 assertTrue("Table should be disabled.",
903 am.getZKTable().isDisabledTable(REGIONINFO.getTable()));
904 } finally {
905 this.server.getConfiguration().setClass(
906 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SimpleLoadBalancer.class,
907 LoadBalancer.class);
908 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
909 am.shutdown();
910 }
911 }
912
913
914
915
916
917
918
919
920 @Test
921 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
922 enabling = true;
923 List<ServerName> destServers = new ArrayList<ServerName>(1);
924 destServers.add(SERVERNAME_A);
925 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
926 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
927 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
928 Server server = new HMaster(HTU.getConfiguration());
929 Whitebox.setInternalState(server, "serverManager", this.serverManager);
930 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
931 this.serverManager);
932 try {
933
934 am.getZKTable().setEnablingTable(REGIONINFO.getTable());
935 new EnableTableHandler(server, REGIONINFO.getTable(),
936 am.getCatalogTracker(), am, new NullTableLockManager(), true).prepare()
937 .process();
938 assertEquals("Number of assignments should be 1.", 1, assignmentCount);
939 assertTrue("Table should be enabled.",
940 am.getZKTable().isEnabledTable(REGIONINFO.getTable()));
941 } finally {
942 enabling = false;
943 assignmentCount = 0;
944 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
945 am.shutdown();
946 ZKAssign.deleteAllNodes(this.watcher);
947 }
948 }
949
950
951
952
953
954
955
956
957 @Test
958 public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
959 throws KeeperException, IOException, Exception {
960 List<ServerName> destServers = new ArrayList<ServerName>(1);
961 destServers.add(SERVERNAME_A);
962 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
963 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
964 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
965 Server server = new HMaster(HTU.getConfiguration());
966 Whitebox.setInternalState(server, "serverManager", this.serverManager);
967 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
968 this.serverManager);
969 try {
970 TableName tableName = TableName.valueOf("dummyTable");
971
972 am.getZKTable().setEnablingTable(tableName);
973 am.joinCluster();
974 assertFalse("Table should not be present in zookeeper.",
975 am.getZKTable().isTablePresent(tableName));
976 } finally {
977 }
978 }
979
980
981
982
983
984 @Test
985 public void testSSHTimesOutOpeningRegionTransition()
986 throws KeeperException, IOException, ServiceException {
987
988 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
989
990 AssignmentManagerWithExtrasForTesting am =
991 setUpMockedAssignmentManager(this.server, this.serverManager);
992
993 RegionState state = new RegionState(REGIONINFO,
994 State.OPENING, System.currentTimeMillis(), SERVERNAME_A);
995 am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_B);
996 am.getRegionStates().regionsInTransition.put(REGIONINFO.getEncodedName(), state);
997
998 am.regionPlans.put(REGIONINFO.getEncodedName(),
999 new RegionPlan(REGIONINFO, SERVERNAME_B, SERVERNAME_A));
1000 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
1001
1002 try {
1003 am.assignInvoked = false;
1004 processServerShutdownHandler(ct, am, false);
1005 assertTrue(am.assignInvoked);
1006 } finally {
1007 am.getRegionStates().regionsInTransition.remove(REGIONINFO.getEncodedName());
1008 am.regionPlans.remove(REGIONINFO.getEncodedName());
1009 }
1010 }
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021 @Test
1022 public void testClosingFailureDuringRecovery() throws Exception {
1023
1024 AssignmentManagerWithExtrasForTesting am =
1025 setUpMockedAssignmentManager(this.server, this.serverManager);
1026 ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_A);
1027 am.getRegionStates().createRegionState(REGIONINFO);
1028
1029 assertFalse( am.getRegionStates().isRegionsInTransition() );
1030
1031 am.processRegionInTransition(REGIONINFO.getEncodedName(), REGIONINFO);
1032
1033 assertTrue( am.getRegionStates().isRegionsInTransition() );
1034 }
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
1053 final HRegionInfo region, final ServerName serverName)
1054 throws KeeperException, IOException {
1055 RegionTransition rt =
1056 RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
1057 region.getRegionName(), serverName);
1058
1059 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1060 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1061 throw new IOException("Failed create of ephemeral " + node);
1062 }
1063
1064
1065 return transitionNodeSplitting(zkw, region, serverName, -1);
1066 }
1067
1068
1069
1070 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
1071 final HRegionInfo parent,
1072 final ServerName serverName, final int version)
1073 throws KeeperException, IOException {
1074 return ZKAssign.transitionNode(zkw, parent, serverName,
1075 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
1076 }
1077
1078 private void unassign(final AssignmentManager am, final ServerName sn,
1079 final HRegionInfo hri) throws RegionException {
1080
1081 am.regionOnline(hri, sn);
1082
1083 am.unassign(hri);
1084 }
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
1096 final ServerManager manager) throws IOException, KeeperException, ServiceException {
1097
1098 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1099
1100
1101
1102
1103
1104 ClientProtos.ClientService.BlockingInterface ri =
1105 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
1106
1107 Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
1108 final ScanResponse.Builder builder = ScanResponse.newBuilder();
1109 builder.setMoreResults(true);
1110 builder.addCellsPerResult(r.size());
1111 final List<CellScannable> rows = new ArrayList<CellScannable>(1);
1112 rows.add(r);
1113 Answer<ScanResponse> ans = new Answer<ClientProtos.ScanResponse>() {
1114 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
1115 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
1116 .getArguments()[0];
1117 if (controller != null) {
1118 controller.setCellScanner(CellUtil.createCellScanner(rows));
1119 }
1120 return builder.build();
1121 }
1122 };
1123 if (enabling) {
1124 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any()))
1125 .thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans)
1126 .thenReturn(ScanResponse.newBuilder().setMoreResults(false).build());
1127 } else {
1128 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any())).thenAnswer(
1129 ans);
1130 }
1131
1132 GetResponse.Builder getBuilder = GetResponse.newBuilder();
1133 getBuilder.setResult(ProtobufUtil.toResult(r));
1134 Mockito.when(ri.get((RpcController)Mockito.any(), (GetRequest) Mockito.any())).
1135 thenReturn(getBuilder.build());
1136
1137 HConnection connection = HConnectionTestingUtility.
1138 getMockedConnectionAndDecorate(HTU.getConfiguration(), null,
1139 ri, SERVERNAME_B, REGIONINFO);
1140
1141 Mockito.when(ct.getConnection()).thenReturn(connection);
1142
1143 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
1144 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
1145 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
1146 server, manager, ct, this.balancer, executor, new NullTableLockManager());
1147 return am;
1148 }
1149
1150
1151
1152
1153 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1154
1155 private final ExecutorService es;
1156
1157 private final CatalogTracker ct;
1158 boolean processRITInvoked = false;
1159 boolean assignInvoked = false;
1160 AtomicBoolean gate = new AtomicBoolean(true);
1161
1162 public AssignmentManagerWithExtrasForTesting(
1163 final Server master, final ServerManager serverManager,
1164 final CatalogTracker catalogTracker, final LoadBalancer balancer,
1165 final ExecutorService service, final TableLockManager tableLockManager)
1166 throws KeeperException, IOException {
1167 super(master, serverManager, catalogTracker, balancer, service, null, tableLockManager);
1168 this.es = service;
1169 this.ct = catalogTracker;
1170 }
1171
1172 @Override
1173 boolean processRegionInTransition(String encodedRegionName,
1174 HRegionInfo regionInfo) throws KeeperException, IOException {
1175 this.processRITInvoked = true;
1176 return super.processRegionInTransition(encodedRegionName, regionInfo);
1177 }
1178
1179 @Override
1180 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan) {
1181 if (enabling) {
1182 assignmentCount++;
1183 this.regionOnline(region, SERVERNAME_A);
1184 } else {
1185 super.assign(region, setOfflineInZK, forceNewPlan);
1186 this.gate.set(true);
1187 }
1188 }
1189
1190 @Override
1191 boolean assign(ServerName destination, List<HRegionInfo> regions) {
1192 if (enabling) {
1193 for (HRegionInfo region : regions) {
1194 assignmentCount++;
1195 this.regionOnline(region, SERVERNAME_A);
1196 }
1197 return true;
1198 }
1199 return super.assign(destination, regions);
1200 }
1201
1202 @Override
1203 public void assign(List<HRegionInfo> regions)
1204 throws IOException, InterruptedException {
1205 assignInvoked = (regions != null && regions.size() > 0);
1206 super.assign(regions);
1207 this.gate.set(true);
1208 }
1209
1210
1211 void setWatcher(ZooKeeperWatcher watcher) {
1212 this.watcher = watcher;
1213 }
1214
1215
1216
1217
1218 ExecutorService getExecutorService() {
1219 return this.es;
1220 }
1221
1222
1223
1224
1225 CatalogTracker getCatalogTracker() {
1226 return this.ct;
1227 }
1228 }
1229
1230
1231
1232
1233
1234
1235
1236 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1237 final ZooKeeperWatcher watcher) {
1238
1239
1240 watcher.registerListenerFirst(am);
1241 Thread t = new Thread("RunAmJoinCluster") {
1242 public void run() {
1243
1244
1245
1246
1247
1248 am.getRegionStates().regionsInTransition.clear();
1249 am.regionPlans.clear();
1250 try {
1251 am.joinCluster();
1252 } catch (IOException e) {
1253 throw new RuntimeException(e);
1254 } catch (KeeperException e) {
1255 throw new RuntimeException(e);
1256 } catch (InterruptedException e) {
1257 throw new RuntimeException(e);
1258 }
1259 }
1260 };
1261 t.start();
1262 while (!t.isAlive()) Threads.sleep(1);
1263 }
1264
1265 @Test
1266 public void testForceAssignMergingRegion() throws Exception {
1267
1268 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1269
1270 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1271 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1272 server.getConfiguration());
1273
1274 AssignmentManager am = new AssignmentManager(this.server,
1275 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
1276 RegionStates regionStates = am.getRegionStates();
1277 try {
1278
1279 regionStates.updateRegionState(hri, RegionState.State.MERGING);
1280
1281 am.assign(hri, true, true);
1282 assertEquals("The region should be still in merging state",
1283 RegionState.State.MERGING, regionStates.getRegionState(hri).getState());
1284 } finally {
1285 am.shutdown();
1286 }
1287 }
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299 @Test
1300 public void testAssignmentEventIgnoredIfNotExpected() throws KeeperException, IOException {
1301
1302 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1303
1304 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1305 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1306 server.getConfiguration());
1307 final AtomicBoolean zkEventProcessed = new AtomicBoolean(false);
1308
1309 AssignmentManager am = new AssignmentManager(this.server,
1310 this.serverManager, ct, balancer, null, null, master.getTableLockManager()) {
1311
1312 @Override
1313 void handleRegion(final RegionTransition rt, int expectedVersion) {
1314 super.handleRegion(rt, expectedVersion);
1315 if (rt != null && Bytes.equals(hri.getRegionName(),
1316 rt.getRegionName()) && rt.getEventType() == EventType.RS_ZK_REGION_OPENING) {
1317 zkEventProcessed.set(true);
1318 }
1319 }
1320 };
1321 try {
1322
1323 am.getRegionStates().regionOffline(hri);
1324 zkEventProcessed.set(false);
1325 this.watcher.registerListenerFirst(am);
1326 assertFalse("The region should not be in transition",
1327 am.getRegionStates().isRegionInTransition(hri));
1328 ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_A);
1329
1330 ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_A);
1331 long startTime = EnvironmentEdgeManager.currentTimeMillis();
1332 while (!zkEventProcessed.get()) {
1333 assertTrue("Timed out in waiting for ZK event to be processed",
1334 EnvironmentEdgeManager.currentTimeMillis() - startTime < 30000);
1335 Threads.sleepWithoutInterrupt(100);
1336 }
1337 assertFalse(am.getRegionStates().isRegionInTransition(hri));
1338 } finally {
1339 am.shutdown();
1340 }
1341 }
1342
1343
1344
1345
1346
1347
1348 @Test
1349 public void testBalanceRegionOfDeletedTable() throws Exception {
1350 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1351 AssignmentManager am = new AssignmentManager(this.server, this.serverManager,
1352 ct, balancer, null, null, master.getTableLockManager());
1353 RegionStates regionStates = am.getRegionStates();
1354 HRegionInfo hri = REGIONINFO;
1355 regionStates.createRegionState(hri);
1356 assertFalse(regionStates.isRegionInTransition(hri));
1357 RegionPlan plan = new RegionPlan(hri, SERVERNAME_A, SERVERNAME_B);
1358
1359 regionStates.tableDeleted(hri.getTable());
1360 am.balance(plan);
1361 assertFalse("The region should not in transition",
1362 regionStates.isRegionInTransition(hri));
1363 }
1364 }