1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.hbase.CellScannable;
34 import org.apache.hadoop.hbase.CellUtil;
35 import org.apache.hadoop.hbase.CoordinatedStateException;
36 import org.apache.hadoop.hbase.CoordinatedStateManager;
37 import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
38 import org.apache.hadoop.hbase.DoNotRetryIOException;
39 import org.apache.hadoop.hbase.HBaseConfiguration;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.MetaMockingUtil;
44 import org.apache.hadoop.hbase.RegionException;
45 import org.apache.hadoop.hbase.RegionTransition;
46 import org.apache.hadoop.hbase.Server;
47 import org.apache.hadoop.hbase.ServerLoad;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.TableName;
50 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
51 import org.apache.hadoop.hbase.client.ClusterConnection;
52 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
53 import org.apache.hadoop.hbase.client.Result;
54 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
55 import org.apache.hadoop.hbase.coordination.OpenRegionCoordination;
56 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
57 import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
58 import org.apache.hadoop.hbase.exceptions.DeserializationException;
59 import org.apache.hadoop.hbase.executor.EventType;
60 import org.apache.hadoop.hbase.executor.ExecutorService;
61 import org.apache.hadoop.hbase.executor.ExecutorType;
62 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
63 import org.apache.hadoop.hbase.master.RegionState.State;
64 import org.apache.hadoop.hbase.master.TableLockManager.NullTableLockManager;
65 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
66 import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
67 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
68 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
69 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
70 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
71 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
72 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
73 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
74 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
75 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
76 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table;
77 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
78 import org.apache.hadoop.hbase.testclassification.MediumTests;
79 import org.apache.hadoop.hbase.util.Bytes;
80 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
81 import org.apache.hadoop.hbase.util.Threads;
82 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
83 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
84 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
85 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
86 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
87 import org.apache.zookeeper.KeeperException;
88 import org.apache.zookeeper.KeeperException.NodeExistsException;
89 import org.apache.zookeeper.Watcher;
90 import org.junit.After;
91 import org.junit.AfterClass;
92 import org.junit.Before;
93 import org.junit.BeforeClass;
94 import org.junit.Test;
95 import org.junit.experimental.categories.Category;
96 import org.mockito.Mockito;
97 import org.mockito.internal.util.reflection.Whitebox;
98 import org.mockito.invocation.InvocationOnMock;
99 import org.mockito.stubbing.Answer;
100
101 import com.google.protobuf.RpcController;
102 import com.google.protobuf.ServiceException;
103
104
105
106
107
108 @Category(MediumTests.class)
109 public class TestAssignmentManager {
110 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
111 private static final ServerName SERVERNAME_A =
112 ServerName.valueOf("example.org", 1234, 5678);
113 private static final ServerName SERVERNAME_B =
114 ServerName.valueOf("example.org", 0, 5678);
115 private static final HRegionInfo REGIONINFO =
116 new HRegionInfo(TableName.valueOf("t"),
117 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
118 private static int assignmentCount;
119 private static boolean enabling = false;
120
121
122 private Server server;
123 private ServerManager serverManager;
124 private ZooKeeperWatcher watcher;
125 private CoordinatedStateManager cp;
126 private MetaTableLocator mtl;
127 private LoadBalancer balancer;
128 private HMaster master;
129 private ClusterConnection connection;
130
131 @BeforeClass
132 public static void beforeClass() throws Exception {
133 HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true);
134 HTU.startMiniZKCluster();
135 }
136
137 @AfterClass
138 public static void afterClass() throws IOException {
139 HTU.shutdownMiniZKCluster();
140 }
141
142 @Before
143 public void before() throws ZooKeeperConnectionException, IOException {
144
145
146
147
148
149
150 this.server = Mockito.mock(Server.class);
151 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
152 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
153 this.watcher =
154 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
155 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
156 Mockito.doThrow(new RuntimeException("Aborted")).
157 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
158
159 cp = new ZkCoordinatedStateManager();
160 cp.initialize(this.server);
161 cp.start();
162
163 mtl = Mockito.mock(MetaTableLocator.class);
164
165 Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
166 Mockito.when(server.getMetaTableLocator()).thenReturn(mtl);
167
168
169 this.connection =
170 (ClusterConnection)HConnectionTestingUtility.getMockedConnection(HTU.getConfiguration());
171
172
173
174 Mockito.when(server.getConnection()).thenReturn(connection);
175
176
177
178 this.serverManager = Mockito.mock(ServerManager.class);
179 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
180 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
181 Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer());
182 final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
183 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
184 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
185 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
186 new ArrayList<ServerName>(onlineServers.keySet()));
187 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
188
189 List<ServerName> avServers = new ArrayList<ServerName>();
190 avServers.addAll(onlineServers.keySet());
191 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
192 Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers);
193
194 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
195 thenReturn(true);
196 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
197 thenReturn(true);
198
199 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)).
200 thenReturn(RegionOpeningState.OPENED);
201 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)).
202 thenReturn(RegionOpeningState.OPENED);
203 this.master = Mockito.mock(HMaster.class);
204
205 Mockito.when(this.master.getServerManager()).thenReturn(serverManager);
206 }
207
208 @After public void after() throws KeeperException, IOException {
209 if (this.watcher != null) {
210
211 ZKAssign.deleteAllNodes(this.watcher);
212 this.watcher.close();
213 this.cp.stop();
214 }
215 if (this.connection != null) this.connection.close();
216 }
217
218
219
220
221
222
223
224
225
226 @Test(timeout = 60000)
227 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
228 throws IOException, KeeperException, InterruptedException, ServiceException,
229 DeserializationException, CoordinatedStateException {
230 AssignmentManagerWithExtrasForTesting am =
231 setUpMockedAssignmentManager(this.server, this.serverManager);
232 try {
233 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
234 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
235 while (!am.processRITInvoked) Thread.sleep(1);
236
237
238
239 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
240
241 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
242
243
244
245
246
247
248 int versionid =
249 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
250 assertNotSame(versionid, -1);
251 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
252
253
254
255 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
256 assertNotSame(-1, versionid);
257
258 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
259 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
260 EventType.RS_ZK_REGION_OPENING, versionid);
261 assertNotSame(-1, versionid);
262
263 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
264 SERVERNAME_B, versionid);
265 assertNotSame(-1, versionid);
266 am.gate.set(false);
267
268 ZKAssign.blockUntilNoRIT(watcher);
269 } finally {
270 am.getExecutorService().shutdown();
271 am.shutdown();
272 }
273 }
274
275 @Test(timeout = 60000)
276 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
277 throws IOException, KeeperException, InterruptedException, ServiceException,
278 DeserializationException, CoordinatedStateException {
279 AssignmentManagerWithExtrasForTesting am =
280 setUpMockedAssignmentManager(this.server, this.serverManager);
281 try {
282 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
283 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
284 while (!am.processRITInvoked) Thread.sleep(1);
285
286
287
288 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
289
290 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
291
292
293
294
295
296
297 int versionid =
298 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
299 assertNotSame(versionid, -1);
300 am.gate.set(false);
301 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
302
303
304
305 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
306 assertNotSame(-1, versionid);
307
308 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
309 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
310 EventType.RS_ZK_REGION_OPENING, versionid);
311 assertNotSame(-1, versionid);
312
313 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
314 SERVERNAME_B, versionid);
315 assertNotSame(-1, versionid);
316
317
318 ZKAssign.blockUntilNoRIT(watcher);
319 } finally {
320 am.getExecutorService().shutdown();
321 am.shutdown();
322 }
323 }
324
325 @Test(timeout = 60000)
326 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
327 throws IOException, KeeperException, InterruptedException, ServiceException,
328 DeserializationException, CoordinatedStateException {
329 AssignmentManagerWithExtrasForTesting am =
330 setUpMockedAssignmentManager(this.server, this.serverManager);
331 try {
332 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
333 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
334 while (!am.processRITInvoked) Thread.sleep(1);
335
336
337
338 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
339
340 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
341
342
343
344
345
346
347 int versionid =
348 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
349 assertNotSame(versionid, -1);
350 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
351
352 am.gate.set(false);
353
354
355 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
356 assertNotSame(-1, versionid);
357
358 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
359 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
360 EventType.RS_ZK_REGION_OPENING, versionid);
361 assertNotSame(-1, versionid);
362
363 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
364 SERVERNAME_B, versionid);
365 assertNotSame(-1, versionid);
366
367 ZKAssign.blockUntilNoRIT(watcher);
368 } finally {
369 am.getExecutorService().shutdown();
370 am.shutdown();
371 }
372 }
373
374 private void createRegionPlanAndBalance(
375 final AssignmentManager am, final ServerName from,
376 final ServerName to, final HRegionInfo hri) throws RegionException {
377
378
379 am.regionOnline(hri, from);
380
381
382 am.balance(new RegionPlan(hri, from, to));
383 }
384
385
386
387
388
389
390
391
392 @Test (timeout=180000)
393 public void testBalance() throws IOException, KeeperException, DeserializationException,
394 InterruptedException, CoordinatedStateException {
395
396
397 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
398
399
400 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
401 .getConfiguration());
402
403 AssignmentManager am = new AssignmentManager(this.server,
404 this.serverManager, balancer, executor, null, master.getTableLockManager());
405 am.failoverCleanupDone.set(true);
406 try {
407
408
409 this.watcher.registerListenerFirst(am);
410
411
412 am.regionOnline(REGIONINFO, SERVERNAME_A);
413
414 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
415 am.balance(plan);
416
417 RegionStates regionStates = am.getRegionStates();
418
419 assertTrue(regionStates.isRegionInTransition(REGIONINFO)
420 && regionStates.isRegionInState(REGIONINFO, State.FAILED_CLOSE));
421
422 regionStates.updateRegionState(REGIONINFO, State.PENDING_CLOSE);
423
424
425
426
427
428
429 int versionid =
430 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
431 assertNotSame(versionid, -1);
432
433
434
435
436
437 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
438
439
440 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
441 assertNotSame(-1, versionid);
442
443 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
444 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
445 EventType.RS_ZK_REGION_OPENING, versionid);
446 assertNotSame(-1, versionid);
447
448 versionid =
449 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
450 assertNotSame(-1, versionid);
451
452 while(regionStates.isRegionInTransition(REGIONINFO)) Threads.sleep(1);
453 } finally {
454 executor.shutdown();
455 am.shutdown();
456
457 ZKAssign.deleteAllNodes(this.watcher);
458 }
459 }
460
461
462
463
464
465
466 @Test (timeout=180000)
467 public void testShutdownHandler()
468 throws KeeperException, IOException, CoordinatedStateException, ServiceException {
469
470
471 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
472
473
474 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
475 this.server, this.serverManager);
476 try {
477 processServerShutdownHandler(am, false);
478 } finally {
479 executor.shutdown();
480 am.shutdown();
481
482 ZKAssign.deleteAllNodes(this.watcher);
483 }
484 }
485
486
487
488
489
490
491
492
493
494
495 @Test (timeout=180000)
496 public void testSSHWhenDisableTableInProgress() throws KeeperException, IOException,
497 CoordinatedStateException, ServiceException {
498 testCaseWithPartiallyDisabledState(Table.State.DISABLING);
499 testCaseWithPartiallyDisabledState(Table.State.DISABLED);
500 }
501
502
503
504
505
506
507
508
509
510 @Test (timeout=180000)
511 public void testSSHWhenSplitRegionInProgress() throws KeeperException, IOException, Exception {
512
513 testCaseWithSplitRegionPartial(true);
514
515 testCaseWithSplitRegionPartial(false);
516 }
517
518 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException,
519 IOException, InterruptedException,
520 CoordinatedStateException, ServiceException {
521
522
523 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
524
525 ZKAssign.deleteAllNodes(this.watcher);
526
527
528 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
529 this.server, this.serverManager);
530
531 am.regionOnline(REGIONINFO, SERVERNAME_A);
532
533 am.getRegionStates().updateRegionState(
534 REGIONINFO, State.SPLITTING, SERVERNAME_A);
535 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
536 Table.State.ENABLED);
537 RegionTransition data = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
538 REGIONINFO.getRegionName(), SERVERNAME_A);
539 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
540
541 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
542
543 try {
544 processServerShutdownHandler(am, regionSplitDone);
545
546
547
548 if (regionSplitDone) {
549 assertFalse("Region state of region in SPLITTING should be removed from rit.",
550 am.getRegionStates().isRegionsInTransition());
551 } else {
552 while (!am.assignInvoked) {
553 Thread.sleep(1);
554 }
555 assertTrue("Assign should be invoked.", am.assignInvoked);
556 }
557 } finally {
558 REGIONINFO.setOffline(false);
559 REGIONINFO.setSplit(false);
560 executor.shutdown();
561 am.shutdown();
562
563 ZKAssign.deleteAllNodes(this.watcher);
564 }
565 }
566
567 private void testCaseWithPartiallyDisabledState(Table.State state) throws KeeperException,
568 IOException, CoordinatedStateException, ServiceException {
569
570
571 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
572 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
573 ZKAssign.deleteAllNodes(this.watcher);
574
575
576 AssignmentManager am = new AssignmentManager(this.server,
577 this.serverManager, balancer, executor, null, master.getTableLockManager());
578
579 am.regionOnline(REGIONINFO, SERVERNAME_A);
580
581 am.getRegionStates().updateRegionState(REGIONINFO, State.PENDING_CLOSE);
582 if (state == Table.State.DISABLING) {
583 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
584 Table.State.DISABLING);
585 } else {
586 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
587 Table.State.DISABLED);
588 }
589 RegionTransition data = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING,
590 REGIONINFO.getRegionName(), SERVERNAME_A);
591
592
593
594 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
595
596 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
597
598 try {
599 processServerShutdownHandler(am, false);
600
601
602 assertTrue("The znode should be deleted.", ZKUtil.checkExists(this.watcher, node) == -1);
603
604
605
606 if (state == Table.State.DISABLED) {
607 assertFalse("Region state of region in pending close should be removed from rit.",
608 am.getRegionStates().isRegionsInTransition());
609 }
610 } finally {
611 am.setEnabledTable(REGIONINFO.getTable());
612 executor.shutdown();
613 am.shutdown();
614
615 ZKAssign.deleteAllNodes(this.watcher);
616 }
617 }
618
619 private void processServerShutdownHandler(AssignmentManager am, boolean splitRegion)
620 throws IOException, ServiceException {
621
622
623 this.watcher.registerListenerFirst(am);
624
625
626
627 ClientProtos.ClientService.BlockingInterface implementation =
628 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
629
630
631 Result r;
632 if (splitRegion) {
633 r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
634 } else {
635 r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
636 }
637
638 final ScanResponse.Builder builder = ScanResponse.newBuilder();
639 builder.setMoreResults(true);
640 builder.addCellsPerResult(r.size());
641 final List<CellScannable> cellScannables = new ArrayList<CellScannable>(1);
642 cellScannables.add(r);
643 Mockito.when(implementation.scan(
644 (RpcController)Mockito.any(), (ScanRequest)Mockito.any())).
645 thenAnswer(new Answer<ScanResponse>() {
646 @Override
647 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
648 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
649 .getArguments()[0];
650 if (controller != null) {
651 controller.setCellScanner(CellUtil.createCellScanner(cellScannables));
652 }
653 return builder.build();
654 }
655 });
656
657
658 ClusterConnection connection =
659 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
660 null, implementation, SERVERNAME_B, REGIONINFO);
661
662
663
664
665
666 Mockito.when(connection.isManaged()).thenReturn(true);
667 try {
668
669
670 Mockito.when(this.server.getConnection()).thenReturn(connection);
671
672
673
674 DeadServer deadServers = new DeadServer();
675 deadServers.add(SERVERNAME_A);
676
677 MasterFileSystem fs = Mockito.mock(MasterFileSystem.class);
678 Mockito.doNothing().when(fs).setLogRecoveryMode();
679 Mockito.when(fs.getLogRecoveryMode()).thenReturn(RecoveryMode.LOG_REPLAY);
680 MasterServices services = Mockito.mock(MasterServices.class);
681 Mockito.when(services.getAssignmentManager()).thenReturn(am);
682 Mockito.when(services.getServerManager()).thenReturn(this.serverManager);
683 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
684 Mockito.when(services.getMasterFileSystem()).thenReturn(fs);
685 Mockito.when(services.getConnection()).thenReturn(connection);
686 ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
687 services, deadServers, SERVERNAME_A, false);
688 am.failoverCleanupDone.set(true);
689 handler.process();
690
691 } finally {
692 if (connection != null) connection.close();
693 }
694 }
695
696
697
698
699
700
701
702 private ExecutorService startupMasterExecutor(final String name) {
703
704 ExecutorService executor = new ExecutorService(name);
705 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
706 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
707 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
708 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
709 return executor;
710 }
711
712 @Test (timeout=180000)
713 public void testUnassignWithSplitAtSameTime() throws KeeperException,
714 IOException, CoordinatedStateException {
715
716 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
717
718
719
720 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
721
722 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
723 .getConfiguration());
724
725 AssignmentManager am = new AssignmentManager(this.server,
726 this.serverManager, balancer, null, null, master.getTableLockManager());
727 try {
728
729 unassign(am, SERVERNAME_A, hri);
730
731 ZKAssign.deleteClosingNode(this.watcher, hri, SERVERNAME_A);
732
733
734
735 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
736
737
738
739
740 unassign(am, SERVERNAME_A, hri);
741
742 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
743 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
744 assertFalse(am.getRegionStates().isRegionInTransition(hri));
745 } finally {
746 am.shutdown();
747 }
748 }
749
750
751
752
753
754
755
756 @Test(timeout = 60000)
757 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
758 throws IOException, KeeperException, CoordinatedStateException,
759 InterruptedException, ServiceException {
760 final RecoverableZooKeeper recoverableZk = Mockito
761 .mock(RecoverableZooKeeper.class);
762 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
763 this.server, this.serverManager);
764 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
765 null) {
766 @Override
767 public RecoverableZooKeeper getRecoverableZooKeeper() {
768 return recoverableZk;
769 }
770 };
771 ((ZooKeeperWatcher) zkw).registerListener(am);
772 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
773 .getChildren("/hbase/region-in-transition", null);
774 am.setWatcher((ZooKeeperWatcher) zkw);
775 try {
776 am.processDeadServersAndRegionsInTransition(null);
777 fail("Expected to abort");
778 } catch (NullPointerException e) {
779 fail("Should not throw NPE");
780 } catch (RuntimeException e) {
781 assertEquals("Aborted", e.getLocalizedMessage());
782 } finally {
783 am.shutdown();
784 }
785 }
786
787
788
789
790 @Test(timeout = 60000)
791 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
792 ServiceException, InterruptedException, CoordinatedStateException {
793 this.server.getConfiguration().setClass(
794 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockedLoadBalancer.class,
795 LoadBalancer.class);
796 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
797 this.server, this.serverManager);
798 try {
799
800
801
802 AtomicBoolean gate = new AtomicBoolean(false);
803 if (balancer instanceof MockedLoadBalancer) {
804 ((MockedLoadBalancer) balancer).setGateVariable(gate);
805 }
806 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
807 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
808 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A,
809 EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
810 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO
811 .getEncodedName());
812 am.getRegionStates().updateRegionState(
813 REGIONINFO, State.OPENING, SERVERNAME_A);
814
815
816 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(
817 REGIONINFO, null, SERVERNAME_A));
818 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
819 List<ServerName> serverList = new ArrayList<ServerName>(2);
820 serverList.add(SERVERNAME_B);
821 Mockito.when(
822 this.serverManager.createDestinationServersList(SERVERNAME_A))
823 .thenReturn(serverList);
824 am.nodeDataChanged(path);
825
826
827 while (!gate.get()) {
828 Thread.sleep(10);
829 }
830
831
832
833 RegionPlan newRegionPlan = am.regionPlans
834 .get(REGIONINFO.getEncodedName());
835 while (newRegionPlan == null) {
836 Thread.sleep(10);
837 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
838 }
839
840
841
842 assertNotSame("Same region plan should not come", regionPlan,
843 newRegionPlan);
844 assertTrue("Destination servers should be different.", !(regionPlan
845 .getDestination().equals(newRegionPlan.getDestination())));
846
847 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
848 } finally {
849 this.server.getConfiguration().setClass(
850 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SimpleLoadBalancer.class,
851 LoadBalancer.class);
852 am.getExecutorService().shutdown();
853 am.shutdown();
854 }
855 }
856
857
858
859
860
861 public static class MockedLoadBalancer extends SimpleLoadBalancer {
862 private AtomicBoolean gate;
863
864 public void setGateVariable(AtomicBoolean gate) {
865 this.gate = gate;
866 }
867
868 @Override
869 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
870 ServerName randomServerName = super.randomAssignment(regionInfo, servers);
871 this.gate.set(true);
872 return randomServerName;
873 }
874
875 @Override
876 public Map<ServerName, List<HRegionInfo>> retainAssignment(
877 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
878 this.gate.set(true);
879 return super.retainAssignment(regions, servers);
880 }
881 }
882
883
884
885
886
887
888 @Test(timeout = 60000)
889 public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException,
890 KeeperException, ServiceException, CoordinatedStateException, InterruptedException {
891 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
892 this.server, this.serverManager);
893 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
894 int version = ZKAssign.getVersion(this.watcher, REGIONINFO);
895 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
896 EventType.RS_ZK_REGION_OPENING, version);
897 RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING,
898 REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY);
899 version = ZKAssign.getVersion(this.watcher, REGIONINFO);
900 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
901 am.getRegionStates().logSplit(SERVERNAME_A);
902 am.getRegionStates().createRegionState(REGIONINFO);
903 am.gate.set(false);
904
905 BaseCoordinatedStateManager cp = new ZkCoordinatedStateManager();
906 cp.initialize(server);
907 cp.start();
908
909 OpenRegionCoordination orc = cp.getOpenRegionCoordination();
910 ZkOpenRegionCoordination.ZkOpenRegionDetails zkOrd =
911 new ZkOpenRegionCoordination.ZkOpenRegionDetails();
912 zkOrd.setServerName(server.getServerName());
913 zkOrd.setVersion(version);
914
915 assertFalse(am.processRegionsInTransition(rt, REGIONINFO, orc, zkOrd));
916 am.getTableStateManager().setTableState(REGIONINFO.getTable(), Table.State.ENABLED);
917 processServerShutdownHandler(am, false);
918
919 while (!am.gate.get()) {
920 Thread.sleep(10);
921 }
922 assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO
923 .getEncodedName()));
924 am.shutdown();
925 }
926
927
928
929
930
931
932
933
934
935 @Test(timeout = 60000)
936 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
937 throws KeeperException, IOException, Exception {
938 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
939 MockedLoadBalancer.class, LoadBalancer.class);
940 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
941 new HashMap<ServerName, ServerLoad>(0));
942 List<ServerName> destServers = new ArrayList<ServerName>(1);
943 destServers.add(SERVERNAME_A);
944 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
945
946 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
947
948 CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
949 HTU.getConfiguration());
950 Server server = new HMaster(HTU.getConfiguration(), csm);
951 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
952 this.serverManager);
953
954 Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
955
956
957
958 Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
959
960 AtomicBoolean gate = new AtomicBoolean(false);
961 if (balancer instanceof MockedLoadBalancer) {
962 ((MockedLoadBalancer) balancer).setGateVariable(gate);
963 }
964 try{
965
966 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
967 Table.State.DISABLING);
968 am.joinCluster();
969
970 assertFalse(
971 "Assign should not be invoked for disabling table regions during clean cluster startup.",
972 gate.get());
973
974 assertTrue("Table should be disabled.",
975 am.getTableStateManager().isTableState(REGIONINFO.getTable(),
976 Table.State.DISABLED));
977 } finally {
978 this.server.getConfiguration().setClass(
979 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SimpleLoadBalancer.class,
980 LoadBalancer.class);
981 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
982 Table.State.ENABLED);
983 am.shutdown();
984 }
985 }
986
987
988
989
990
991
992
993
994 @Test (timeout=180000)
995 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
996 enabling = true;
997 List<ServerName> destServers = new ArrayList<ServerName>(1);
998 destServers.add(SERVERNAME_A);
999 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
1000 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1001 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1002 CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
1003 HTU.getConfiguration());
1004 Server server = new HMaster(HTU.getConfiguration(), csm);
1005 Whitebox.setInternalState(server, "serverManager", this.serverManager);
1006 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1007 this.serverManager);
1008
1009 Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
1010
1011
1012
1013 Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
1014
1015 try {
1016
1017 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1018 Table.State.ENABLING);
1019 new EnableTableHandler(server, REGIONINFO.getTable(),
1020 am, new NullTableLockManager(), true).prepare()
1021 .process();
1022 assertEquals("Number of assignments should be 1.", 1, assignmentCount);
1023 assertTrue("Table should be enabled.",
1024 am.getTableStateManager().isTableState(REGIONINFO.getTable(),
1025 Table.State.ENABLED));
1026 } finally {
1027 enabling = false;
1028 assignmentCount = 0;
1029 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1030 Table.State.ENABLED);
1031 am.shutdown();
1032 ZKAssign.deleteAllNodes(this.watcher);
1033 }
1034 }
1035
1036
1037
1038
1039
1040
1041
1042
1043 @Test (timeout=180000)
1044 public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
1045 throws Exception {
1046 List<ServerName> destServers = new ArrayList<ServerName>(1);
1047 destServers.add(SERVERNAME_A);
1048 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
1049 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1050 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1051 CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
1052 HTU.getConfiguration());
1053 Server server = new HMaster(HTU.getConfiguration(), csm);
1054 Whitebox.setInternalState(server, "serverManager", this.serverManager);
1055 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1056 this.serverManager);
1057
1058 Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
1059
1060
1061
1062 Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
1063
1064 try {
1065 TableName tableName = TableName.valueOf("dummyTable");
1066
1067 am.getTableStateManager().setTableState(tableName,
1068 Table.State.ENABLING);
1069 am.joinCluster();
1070 assertFalse("Table should not be present in zookeeper.",
1071 am.getTableStateManager().isTablePresent(tableName));
1072 } finally {
1073 am.shutdown();
1074 }
1075 }
1076
1077
1078
1079
1080
1081 @Test (timeout=180000)
1082 public void testSSHTimesOutOpeningRegionTransition()
1083 throws KeeperException, IOException, CoordinatedStateException, ServiceException {
1084
1085 AssignmentManagerWithExtrasForTesting am =
1086 setUpMockedAssignmentManager(this.server, this.serverManager);
1087
1088 RegionState state = new RegionState(REGIONINFO,
1089 State.OPENING, System.currentTimeMillis(), SERVERNAME_A);
1090 am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_B);
1091 am.getRegionStates().regionsInTransition.put(REGIONINFO.getEncodedName(), state);
1092
1093 am.regionPlans.put(REGIONINFO.getEncodedName(),
1094 new RegionPlan(REGIONINFO, SERVERNAME_B, SERVERNAME_A));
1095 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1096 Table.State.ENABLED);
1097
1098 try {
1099 am.assignInvoked = false;
1100 processServerShutdownHandler(am, false);
1101 assertTrue(am.assignInvoked);
1102 } finally {
1103 am.getRegionStates().regionsInTransition.remove(REGIONINFO.getEncodedName());
1104 am.regionPlans.remove(REGIONINFO.getEncodedName());
1105 am.shutdown();
1106 }
1107 }
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118 @Test (timeout=180000)
1119 public void testClosingFailureDuringRecovery() throws Exception {
1120
1121 AssignmentManagerWithExtrasForTesting am =
1122 setUpMockedAssignmentManager(this.server, this.serverManager);
1123 ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_A);
1124 try {
1125 am.getRegionStates().createRegionState(REGIONINFO);
1126
1127 assertFalse( am.getRegionStates().isRegionsInTransition() );
1128
1129 am.processRegionInTransition(REGIONINFO.getEncodedName(), REGIONINFO);
1130
1131 assertTrue( am.getRegionStates().isRegionsInTransition() );
1132 } finally {
1133 am.shutdown();
1134 }
1135 }
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
1154 final HRegionInfo region, final ServerName serverName)
1155 throws KeeperException, IOException {
1156 RegionTransition rt =
1157 RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
1158 region.getRegionName(), serverName);
1159
1160 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1161 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1162 throw new IOException("Failed create of ephemeral " + node);
1163 }
1164
1165
1166 return transitionNodeSplitting(zkw, region, serverName, -1);
1167 }
1168
1169
1170
1171 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
1172 final HRegionInfo parent,
1173 final ServerName serverName, final int version)
1174 throws KeeperException, IOException {
1175 return ZKAssign.transitionNode(zkw, parent, serverName,
1176 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
1177 }
1178
1179 private void unassign(final AssignmentManager am, final ServerName sn,
1180 final HRegionInfo hri) throws RegionException {
1181
1182 am.regionOnline(hri, sn);
1183
1184 am.unassign(hri);
1185 }
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
1197 final ServerManager manager) throws IOException, KeeperException,
1198 ServiceException, CoordinatedStateException {
1199
1200
1201
1202
1203
1204 ClientProtos.ClientService.BlockingInterface ri =
1205 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
1206
1207 Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
1208 final ScanResponse.Builder builder = ScanResponse.newBuilder();
1209 builder.setMoreResults(true);
1210 builder.addCellsPerResult(r.size());
1211 final List<CellScannable> rows = new ArrayList<CellScannable>(1);
1212 rows.add(r);
1213 Answer<ScanResponse> ans = new Answer<ClientProtos.ScanResponse>() {
1214 @Override
1215 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
1216 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
1217 .getArguments()[0];
1218 if (controller != null) {
1219 controller.setCellScanner(CellUtil.createCellScanner(rows));
1220 }
1221 return builder.build();
1222 }
1223 };
1224 if (enabling) {
1225 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any()))
1226 .thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans)
1227 .thenReturn(ScanResponse.newBuilder().setMoreResults(false).build());
1228 } else {
1229 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any())).thenAnswer(
1230 ans);
1231 }
1232
1233 GetResponse.Builder getBuilder = GetResponse.newBuilder();
1234 getBuilder.setResult(ProtobufUtil.toResult(r));
1235 Mockito.when(ri.get((RpcController)Mockito.any(), (GetRequest) Mockito.any())).
1236 thenReturn(getBuilder.build());
1237
1238 ClusterConnection connection = (ClusterConnection)HConnectionTestingUtility.
1239 getMockedConnectionAndDecorate(HTU.getConfiguration(), null,
1240 ri, SERVERNAME_B, REGIONINFO);
1241
1242
1243
1244
1245
1246 Mockito.when(connection.isManaged()).thenReturn(true);
1247
1248
1249 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
1250 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
1251 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
1252 server, connection, manager, this.balancer, executor, new NullTableLockManager());
1253 return am;
1254 }
1255
1256
1257
1258
1259 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1260
1261 private final ExecutorService es;
1262 boolean processRITInvoked = false;
1263 boolean assignInvoked = false;
1264 AtomicBoolean gate = new AtomicBoolean(true);
1265 private ClusterConnection connection;
1266
1267 public AssignmentManagerWithExtrasForTesting(
1268 final Server master, ClusterConnection connection, final ServerManager serverManager,
1269 final LoadBalancer balancer,
1270 final ExecutorService service, final TableLockManager tableLockManager)
1271 throws KeeperException, IOException, CoordinatedStateException {
1272 super(master, serverManager, balancer, service, null, tableLockManager);
1273 this.es = service;
1274 this.connection = connection;
1275 }
1276
1277 @Override
1278 boolean processRegionInTransition(String encodedRegionName,
1279 HRegionInfo regionInfo) throws KeeperException, IOException {
1280 this.processRITInvoked = true;
1281 return super.processRegionInTransition(encodedRegionName, regionInfo);
1282 }
1283
1284 @Override
1285 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan) {
1286 if (enabling) {
1287 assignmentCount++;
1288 this.regionOnline(region, SERVERNAME_A);
1289 } else {
1290 super.assign(region, setOfflineInZK, forceNewPlan);
1291 this.gate.set(true);
1292 }
1293 }
1294
1295 @Override
1296 boolean assign(ServerName destination, List<HRegionInfo> regions)
1297 throws InterruptedException {
1298 if (enabling) {
1299 for (HRegionInfo region : regions) {
1300 assignmentCount++;
1301 this.regionOnline(region, SERVERNAME_A);
1302 }
1303 return true;
1304 }
1305 return super.assign(destination, regions);
1306 }
1307
1308 @Override
1309 public void assign(List<HRegionInfo> regions)
1310 throws IOException, InterruptedException {
1311 assignInvoked = (regions != null && regions.size() > 0);
1312 super.assign(regions);
1313 this.gate.set(true);
1314 }
1315
1316
1317 void setWatcher(ZooKeeperWatcher watcher) {
1318 this.watcher = watcher;
1319 }
1320
1321
1322
1323
1324 ExecutorService getExecutorService() {
1325 return this.es;
1326 }
1327
1328
1329
1330
1331 ClusterConnection getConnection() {
1332 return this.connection;
1333 }
1334
1335 @Override
1336 public void shutdown() {
1337 super.shutdown();
1338 if (this.connection != null)
1339 try {
1340 this.connection.close();
1341 } catch (IOException e) {
1342 fail("Failed to close connection");
1343 }
1344 }
1345 }
1346
1347
1348
1349
1350
1351
1352
1353 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1354 final ZooKeeperWatcher watcher) {
1355
1356
1357 watcher.registerListenerFirst(am);
1358 Thread t = new Thread("RunAmJoinCluster") {
1359 @Override
1360 public void run() {
1361
1362
1363
1364
1365
1366 am.getRegionStates().regionsInTransition.clear();
1367 am.regionPlans.clear();
1368 try {
1369 am.joinCluster();
1370 } catch (IOException e) {
1371 throw new RuntimeException(e);
1372 } catch (KeeperException e) {
1373 throw new RuntimeException(e);
1374 } catch (InterruptedException e) {
1375 throw new RuntimeException(e);
1376 } catch (CoordinatedStateException e) {
1377 throw new RuntimeException(e);
1378 }
1379 }
1380 };
1381 t.start();
1382 while (!t.isAlive()) Threads.sleep(1);
1383 }
1384
1385 @Test (timeout=180000)
1386 public void testForceAssignMergingRegion() throws Exception {
1387
1388 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1389
1390 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1391 server.getConfiguration());
1392
1393 AssignmentManager am = new AssignmentManager(this.server,
1394 this.serverManager, balancer, null, null, master.getTableLockManager());
1395 RegionStates regionStates = am.getRegionStates();
1396 try {
1397
1398 regionStates.updateRegionState(hri, RegionState.State.MERGING);
1399
1400 am.assign(hri, true, true);
1401 assertEquals("The region should be still in merging state",
1402 RegionState.State.MERGING, regionStates.getRegionState(hri).getState());
1403 } finally {
1404 am.shutdown();
1405 }
1406 }
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418 @Test (timeout=180000)
1419 public void testAssignmentEventIgnoredIfNotExpected() throws KeeperException, IOException,
1420 CoordinatedStateException {
1421
1422 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1423 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1424 server.getConfiguration());
1425 final AtomicBoolean zkEventProcessed = new AtomicBoolean(false);
1426
1427 AssignmentManager am = new AssignmentManager(this.server,
1428 this.serverManager, balancer, null, null, master.getTableLockManager()) {
1429
1430 @Override
1431 void handleRegion(final RegionTransition rt, OpenRegionCoordination coordination,
1432 OpenRegionCoordination.OpenRegionDetails ord) {
1433 super.handleRegion(rt, coordination, ord);
1434 if (rt != null && Bytes.equals(hri.getRegionName(),
1435 rt.getRegionName()) && rt.getEventType() == EventType.RS_ZK_REGION_OPENING) {
1436 zkEventProcessed.set(true);
1437 }
1438 }
1439 };
1440 try {
1441
1442 am.getRegionStates().regionOffline(hri);
1443 zkEventProcessed.set(false);
1444 this.watcher.registerListenerFirst(am);
1445 assertFalse("The region should not be in transition",
1446 am.getRegionStates().isRegionInTransition(hri));
1447 ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_A);
1448
1449 ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_A);
1450 long startTime = EnvironmentEdgeManager.currentTime();
1451 while (!zkEventProcessed.get()) {
1452 assertTrue("Timed out in waiting for ZK event to be processed",
1453 EnvironmentEdgeManager.currentTime() - startTime < 30000);
1454 Threads.sleepWithoutInterrupt(100);
1455 }
1456 assertFalse(am.getRegionStates().isRegionInTransition(hri));
1457 } finally {
1458 am.shutdown();
1459 }
1460 }
1461
1462
1463
1464
1465
1466
1467 @Test (timeout=180000)
1468 public void testBalanceRegionOfDeletedTable() throws Exception {
1469 AssignmentManager am = new AssignmentManager(this.server, this.serverManager,
1470 balancer, null, null, master.getTableLockManager());
1471 RegionStates regionStates = am.getRegionStates();
1472 HRegionInfo hri = REGIONINFO;
1473 regionStates.createRegionState(hri);
1474 assertFalse(regionStates.isRegionInTransition(hri));
1475 RegionPlan plan = new RegionPlan(hri, SERVERNAME_A, SERVERNAME_B);
1476
1477 regionStates.tableDeleted(hri.getTable());
1478 am.balance(plan);
1479 assertFalse("The region should not in transition",
1480 regionStates.isRegionInTransition(hri));
1481 am.shutdown();
1482 }
1483
1484
1485
1486
1487
1488 @SuppressWarnings("unchecked")
1489 @Test (timeout=180000)
1490 public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
1491 Mockito.when(this.serverManager.sendRegionOpen(Mockito.eq(SERVERNAME_B), Mockito.eq(REGIONINFO),
1492 Mockito.anyInt(), (List<ServerName>)Mockito.any()))
1493 .thenThrow(new DoNotRetryIOException());
1494 this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 100);
1495
1496 HRegionInfo hri = REGIONINFO;
1497 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1498 server.getConfiguration());
1499
1500 AssignmentManager am = new AssignmentManager(this.server,
1501 this.serverManager, balancer, null, null, master.getTableLockManager());
1502 RegionStates regionStates = am.getRegionStates();
1503 try {
1504 am.regionPlans.put(REGIONINFO.getEncodedName(),
1505 new RegionPlan(REGIONINFO, null, SERVERNAME_B));
1506
1507
1508 am.assign(hri, true, false);
1509 } finally {
1510 assertEquals(SERVERNAME_A, regionStates.getRegionState(REGIONINFO).getServerName());
1511 am.shutdown();
1512 }
1513 }
1514 }