1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNotSame;
25 import static org.junit.Assert.assertNull;
26 import static org.junit.Assert.assertTrue;
27 import static org.junit.Assert.fail;
28
29 import java.io.IOException;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.concurrent.CountDownLatch;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.Abortable;
38 import org.apache.hadoop.hbase.HBaseIOException;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.LargeTests;
45 import org.apache.hadoop.hbase.MasterNotRunningException;
46 import org.apache.hadoop.hbase.MiniHBaseCluster;
47 import org.apache.hadoop.hbase.RegionTransition;
48 import org.apache.hadoop.hbase.Server;
49 import org.apache.hadoop.hbase.ServerName;
50 import org.apache.hadoop.hbase.TableName;
51 import org.apache.hadoop.hbase.UnknownRegionException;
52 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
53 import org.apache.hadoop.hbase.catalog.MetaReader;
54 import org.apache.hadoop.hbase.client.Delete;
55 import org.apache.hadoop.hbase.client.HBaseAdmin;
56 import org.apache.hadoop.hbase.client.HTable;
57 import org.apache.hadoop.hbase.client.Put;
58 import org.apache.hadoop.hbase.client.Result;
59 import org.apache.hadoop.hbase.client.ResultScanner;
60 import org.apache.hadoop.hbase.client.Scan;
61 import org.apache.hadoop.hbase.exceptions.DeserializationException;
62 import org.apache.hadoop.hbase.executor.EventType;
63 import org.apache.hadoop.hbase.master.AssignmentManager;
64 import org.apache.hadoop.hbase.master.HMaster;
65 import org.apache.hadoop.hbase.master.RegionState;
66 import org.apache.hadoop.hbase.master.RegionState.State;
67 import org.apache.hadoop.hbase.master.RegionStates;
68 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
69 import org.apache.hadoop.hbase.util.Bytes;
70 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
71 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
72 import org.apache.hadoop.hbase.util.PairOfSameType;
73 import org.apache.hadoop.hbase.util.Threads;
74 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
75 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
76 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
77 import org.apache.zookeeper.KeeperException;
78 import org.apache.zookeeper.KeeperException.NodeExistsException;
79 import org.apache.zookeeper.data.Stat;
80 import org.junit.After;
81 import org.junit.AfterClass;
82 import org.junit.Assert;
83 import org.junit.Before;
84 import org.junit.BeforeClass;
85 import org.junit.Test;
86 import org.junit.experimental.categories.Category;
87
88 import com.google.protobuf.ServiceException;
89
90
91
92
93
94
95 @Category(LargeTests.class)
96 public class TestSplitTransactionOnCluster {
97 private static final Log LOG =
98 LogFactory.getLog(TestSplitTransactionOnCluster.class);
99 private HBaseAdmin admin = null;
100 private MiniHBaseCluster cluster = null;
101 private static final int NB_SERVERS = 3;
102 private static CountDownLatch latch = new CountDownLatch(1);
103 private static volatile boolean secondSplit = false;
104 private static volatile boolean callRollBack = false;
105 private static volatile boolean firstSplitCompleted = false;
106 private static byte [] CF = Bytes.toBytes("cf");
107
108 private static final HBaseTestingUtility TESTING_UTIL =
109 new HBaseTestingUtility();
110
111 @BeforeClass public static void before() throws Exception {
112 TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
113
114
115 TESTING_UTIL.getConfiguration().setInt(
116 "hbase.master.assignment.timeoutmonitor.timeout", 4000);
117 TESTING_UTIL.startMiniCluster(NB_SERVERS);
118 }
119
120 @AfterClass public static void after() throws Exception {
121 TESTING_UTIL.shutdownMiniCluster();
122 }
123
124 @Before public void setup() throws IOException {
125 TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
126 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
127 this.cluster = TESTING_UTIL.getMiniHBaseCluster();
128 }
129
130 @After
131 public void tearDown() throws Exception {
132 this.admin.close();
133 }
134
135 private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
136 assertEquals(1, regions.size());
137 HRegionInfo hri = regions.get(0).getRegionInfo();
138 return waitOnRIT(hri);
139 }
140
141
142
143
144
145
146
147
148 private HRegionInfo waitOnRIT(final HRegionInfo hri) {
149
150
151 while (TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
152 getRegionStates().isRegionInTransition(hri)) {
153 LOG.info("Waiting on region in transition: " +
154 TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().
155 getRegionTransitionState(hri));
156 Threads.sleep(10);
157 }
158 return hri;
159 }
160
161 @SuppressWarnings("deprecation")
162 @Test(timeout = 60000)
163 public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
164 final TableName tableName =
165 TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
166 try {
167
168 HTable t = createTableAndWait(tableName.getName(), CF);
169 final List<HRegion> regions = cluster.getRegions(tableName);
170 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
171 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
172 final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
173 insertData(tableName.getName(), admin, t);
174 t.close();
175
176
177 this.admin.setBalancerRunning(false, true);
178
179 cluster.getMaster().setCatalogJanitorEnabled(false);
180
181
182 final HRegion region = findSplittableRegion(regions);
183 assertTrue("not able to find a splittable region", region != null);
184
185 new Thread() {
186 public void run() {
187 SplitTransaction st = null;
188 st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
189 try {
190 st.prepare();
191 st.execute(regionServer, regionServer);
192 } catch (IOException e) {
193
194 }
195 }
196 }.start();
197 for (int i = 0; !callRollBack && i < 100; i++) {
198 Thread.sleep(100);
199 }
200 assertTrue("Waited too long for rollback", callRollBack);
201 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
202 try {
203 secondSplit = true;
204
205 region.initialize();
206 st.prepare();
207 st.execute(regionServer, regionServer);
208 } catch (IOException e) {
209 LOG.debug("Rollback started :"+ e.getMessage());
210 st.rollback(regionServer, regionServer);
211 }
212 for (int i=0; !firstSplitCompleted && i<100; i++) {
213 Thread.sleep(100);
214 }
215 assertTrue("fist split did not complete", firstSplitCompleted);
216
217 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
218 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
219
220 for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
221 Thread.sleep(100);
222 }
223 assertFalse("region still in transition", rit.containsKey(
224 rit.containsKey(hri.getTable())));
225
226 List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
227
228 assertEquals("The parent region should be splitted", 2, onlineRegions.size());
229
230 List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
231 .getRegionStates().getRegionsOfTable(tableName);
232
233 assertEquals("No of regions in master", 2, regionsOfTable.size());
234 } finally {
235 admin.setBalancerRunning(true, false);
236 secondSplit = false;
237 firstSplitCompleted = false;
238 callRollBack = false;
239 cluster.getMaster().setCatalogJanitorEnabled(true);
240 TESTING_UTIL.deleteTable(tableName);
241 }
242 }
243
244
245
246
247
248
249
250
251
252
253
254
255 @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
256 throws IOException, InterruptedException, NodeExistsException, KeeperException,
257 DeserializationException, ServiceException {
258 final byte [] tableName =
259 Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
260
261
262 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
263 List<HRegion> regions = cluster.getRegions(tableName);
264 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
265
266 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
267
268
269 this.admin.setBalancerRunning(false, true);
270
271 cluster.getMaster().setCatalogJanitorEnabled(false);
272 try {
273
274 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
275
276 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
277 printOutRegions(server, "Initial regions: ");
278 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
279
280
281 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
282
283 split(hri, server, regionCount);
284
285 List<HRegion> daughters = checkAndGetDaughters(tableName);
286
287 String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
288 hri.getEncodedName());
289 RegionTransition rt = null;
290 Stat stats = null;
291
292 for (int i=0; i<100; i++) {
293 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
294 rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
295 hri.getEncodedName()));
296 if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
297 Thread.sleep(100);
298 }
299 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
300 assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
301
302 cluster.abortRegionServer(tableRegionIndex);
303 waitUntilRegionServerDead();
304 awaitDaughters(tableName, daughters.size());
305
306
307 regions = cluster.getRegions(tableName);
308 for (HRegion r: regions) {
309 assertTrue(daughters.contains(r));
310 }
311
312 for (int i=0; i<100; i++) {
313
314 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
315 if (stats == null) break;
316 Thread.sleep(100);
317 }
318 LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
319 assertTrue(stats == null);
320 } finally {
321
322 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
323 admin.setBalancerRunning(true, false);
324 cluster.getMaster().setCatalogJanitorEnabled(true);
325 t.close();
326 }
327 }
328
329 @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
330 throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
331 final byte [] tableName =
332 Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
333
334
335 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
336 List<HRegion> regions = cluster.getRegions(tableName);
337 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
338
339 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
340
341
342 this.admin.setBalancerRunning(false, true);
343
344 cluster.getMaster().setCatalogJanitorEnabled(false);
345 try {
346
347 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
348
349 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
350 printOutRegions(server, "Initial regions: ");
351 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
352
353
354 ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
355 ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
356 hri, fakedServer);
357
358
359 this.admin.split(hri.getRegionNameAsString());
360 this.admin.split(hri.getRegionNameAsString());
361 this.admin.split(hri.getRegionNameAsString());
362
363 for (int i = 0; i < 10; i++) {
364 Thread.sleep(100);
365 assertEquals(regionCount, ProtobufUtil.getOnlineRegions(server).size());
366 }
367
368 ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
369 hri, fakedServer);
370
371 split(hri, server, regionCount);
372
373 checkAndGetDaughters(tableName);
374
375 } finally {
376 admin.setBalancerRunning(true, false);
377 cluster.getMaster().setCatalogJanitorEnabled(true);
378 t.close();
379 }
380 }
381
382
383
384
385
386
387
388 @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
389 throws IOException, InterruptedException, ServiceException {
390 final byte [] tableName =
391 Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
392
393
394 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
395 List<HRegion> regions = cluster.getRegions(tableName);
396 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
397
398 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
399
400
401 this.admin.setBalancerRunning(false, true);
402
403 cluster.getMaster().setCatalogJanitorEnabled(false);
404 try {
405
406 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
407
408 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
409 printOutRegions(server, "Initial regions: ");
410 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
411
412 split(hri, server, regionCount);
413
414 List<HRegion> daughters = checkAndGetDaughters(tableName);
415
416 regionCount = ProtobufUtil.getOnlineRegions(server).size();
417 HRegionInfo daughter = daughters.get(0).getRegionInfo();
418 LOG.info("Daughter we are going to split: " + daughter);
419
420
421 this.admin.compact(daughter.getRegionName());
422 daughters = cluster.getRegions(tableName);
423 HRegion daughterRegion = null;
424 for (HRegion r: daughters) {
425 if (r.getRegionInfo().equals(daughter)) {
426 daughterRegion = r;
427 LOG.info("Found matching HRI: " + daughterRegion);
428 break;
429 }
430 }
431 assertTrue(daughterRegion != null);
432 for (int i=0; i<100; i++) {
433 if (!daughterRegion.hasReferences()) break;
434 Threads.sleep(100);
435 }
436 assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
437 LOG.info("Daughter hri before split (has been compacted): " + daughter);
438 split(daughter, server, regionCount);
439
440 daughters = cluster.getRegions(tableName);
441 for (HRegion d: daughters) {
442 LOG.info("Regions before crash: " + d);
443 }
444
445 cluster.abortRegionServer(tableRegionIndex);
446 waitUntilRegionServerDead();
447 awaitDaughters(tableName, daughters.size());
448
449
450 regions = cluster.getRegions(tableName);
451 for (HRegion d: daughters) {
452 LOG.info("Regions after crash: " + d);
453 }
454 assertEquals(daughters.size(), regions.size());
455 for (HRegion r: regions) {
456 LOG.info("Regions post crash " + r);
457 assertTrue("Missing region post crash " + r, daughters.contains(r));
458 }
459 } finally {
460 admin.setBalancerRunning(true, false);
461 cluster.getMaster().setCatalogJanitorEnabled(true);
462 t.close();
463 }
464 }
465
466 @Test(timeout = 180000)
467 public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
468 Configuration conf = TESTING_UTIL.getConfiguration();
469 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
470 TableName userTableName =
471 TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
472 HTableDescriptor htd = new HTableDescriptor(userTableName);
473 HColumnDescriptor hcd = new HColumnDescriptor("col");
474 htd.addFamily(hcd);
475 admin.createTable(htd);
476 ZKAssign.blockUntilNoRIT(zkw);
477 HTable table = new HTable(conf, userTableName);
478 try {
479 for (int i = 0; i <= 5; i++) {
480 String row = "row" + i;
481 Put p = new Put(row.getBytes());
482 String val = "Val" + i;
483 p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
484 table.put(p);
485 admin.flush(userTableName.getName());
486 Delete d = new Delete(row.getBytes());
487
488 table.delete(d);
489 admin.flush(userTableName.getName());
490 }
491 admin.majorCompact(userTableName.getName());
492 List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
493 .getMaster().getAssignmentManager().getRegionStates()
494 .getRegionsOfTable(userTableName);
495 HRegionInfo hRegionInfo = regionsOfTable.get(0);
496 Put p = new Put("row6".getBytes());
497 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
498 table.put(p);
499 p = new Put("row7".getBytes());
500 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
501 table.put(p);
502 p = new Put("row8".getBytes());
503 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
504 table.put(p);
505 admin.flush(userTableName.getName());
506 admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
507 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
508 .getAssignmentManager().getRegionStates()
509 .getRegionsOfTable(userTableName);
510
511 while (regionsOfTable.size() != 2) {
512 Thread.sleep(2000);
513 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
514 .getAssignmentManager().getRegionStates()
515 .getRegionsOfTable(userTableName);
516 }
517 Assert.assertEquals(2, regionsOfTable.size());
518 Scan s = new Scan();
519 ResultScanner scanner = table.getScanner(s);
520 int mainTableCount = 0;
521 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
522 mainTableCount++;
523 }
524 Assert.assertEquals(3, mainTableCount);
525 } finally {
526 table.close();
527 }
528 }
529
530
531
532
533 static class UselessTestAbortable implements Abortable {
534 boolean aborted = false;
535 @Override
536 public void abort(String why, Throwable e) {
537 LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
538 aborted = true;
539 }
540
541 @Override
542 public boolean isAborted() {
543 return this.aborted;
544 }
545 }
546
547
548
549
550
551
552
553
554
555
556
557 @Test(timeout = 400000)
558 public void testMasterRestartWhenSplittingIsPartial()
559 throws IOException, InterruptedException, NodeExistsException,
560 KeeperException, DeserializationException, ServiceException {
561 final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
562
563
564 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
565 List<HRegion> regions = cluster.getRegions(tableName);
566 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
567
568 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
569
570
571 this.admin.setBalancerRunning(false, true);
572
573 cluster.getMaster().setCatalogJanitorEnabled(false);
574 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
575 "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
576 try {
577
578 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
579
580 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
581 printOutRegions(server, "Initial regions: ");
582
583
584 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
585
586
587 this.admin.split(hri.getRegionNameAsString());
588 checkAndGetDaughters(tableName);
589
590 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
591 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
592 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
593 + stats);
594 byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
595 RegionTransition rtd = RegionTransition.parseFrom(bytes);
596
597 assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
598 || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
599
600
601 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
602
603 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
604
605
606
607 hri.setOffline(true);
608 hri.setSplit(true);
609 ServerName regionServerOfRegion = master.getAssignmentManager()
610 .getRegionStates().getRegionServerOfRegion(hri);
611 assertTrue(regionServerOfRegion != null);
612
613
614 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
615 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
616 Stat stat = new Stat();
617 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
618
619 for (int i=0; data != null && i<60; i++) {
620 Thread.sleep(1000);
621 data = ZKUtil.getDataNoWatch(zkw, node, stat);
622 }
623 assertNull("Waited too long for ZK node to be removed: "+node, data);
624 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
625 assertTrue("Split parent should be in SPLIT state",
626 regionStates.isRegionInState(hri, State.SPLIT));
627 regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
628 assertTrue(regionServerOfRegion == null);
629 } finally {
630
631 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
632 admin.setBalancerRunning(true, false);
633 cluster.getMaster().setCatalogJanitorEnabled(true);
634 t.close();
635 zkw.close();
636 }
637 }
638
639
640
641
642
643
644
645
646
647 @Test (timeout = 300000)
648 public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
649 throws IOException, InterruptedException, NodeExistsException,
650 KeeperException, ServiceException {
651 final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
652
653
654 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
655 List<HRegion> regions = cluster.getRegions(tableName);
656 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
657
658 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
659
660
661 this.admin.setBalancerRunning(false, true);
662
663 cluster.getMaster().setCatalogJanitorEnabled(false);
664 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
665 "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
666 try {
667
668 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
669
670 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
671 printOutRegions(server, "Initial regions: ");
672
673 this.admin.split(hri.getRegionNameAsString());
674 checkAndGetDaughters(tableName);
675
676 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
677 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
678 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
679 + stats);
680 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
681 Stat stat = new Stat();
682 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
683
684 for (int i=0; data != null && i<60; i++) {
685 Thread.sleep(1000);
686 data = ZKUtil.getDataNoWatch(zkw, node, stat);
687 }
688 assertNull("Waited too long for ZK node to be removed: "+node, data);
689
690 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
691
692 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
693
694
695
696 hri.setOffline(true);
697 hri.setSplit(true);
698 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
699 assertTrue("Split parent should be in SPLIT state",
700 regionStates.isRegionInState(hri, State.SPLIT));
701 ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
702 assertTrue(regionServerOfRegion == null);
703 } finally {
704 this.admin.setBalancerRunning(true, false);
705 cluster.getMaster().setCatalogJanitorEnabled(true);
706 t.close();
707 zkw.close();
708 }
709 }
710
711
712
713
714
715
716
717
718
719
720
721
722 @Test
723 public void testSplitBeforeSettingSplittingInZK() throws Exception,
724 InterruptedException, KeeperException {
725 testSplitBeforeSettingSplittingInZKInternals();
726 }
727
728 @Test(timeout = 60000)
729 public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
730 final TableName tableName =
731 TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
732
733 HTable t = createTableAndWait(tableName.getName(), CF);
734 try {
735 List<HRegion> regions = cluster.getRegions(tableName);
736 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
737 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
738 insertData(tableName.getName(), admin, t);
739
740 admin.setBalancerRunning(false, true);
741
742 cluster.getMaster().setCatalogJanitorEnabled(false);
743 boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
744 tableName);
745 assertEquals("The specified table should present.", true, tableExists);
746 final HRegion region = findSplittableRegion(regions);
747 assertTrue("not able to find a splittable region", region != null);
748 SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
749 try {
750 st.prepare();
751 st.createDaughters(regionServer, regionServer);
752 } catch (IOException e) {
753
754 }
755 tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
756 tableName);
757 assertEquals("The specified table should present.", true, tableExists);
758 } finally {
759 admin.setBalancerRunning(true, false);
760 cluster.getMaster().setCatalogJanitorEnabled(true);
761 t.close();
762 }
763 }
764
765 private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
766 InterruptedException {
767 Put p = new Put(Bytes.toBytes("row1"));
768 p.add(CF, Bytes.toBytes("q1"), Bytes.toBytes("1"));
769 t.put(p);
770 p = new Put(Bytes.toBytes("row2"));
771 p.add(CF, Bytes.toBytes("q1"), Bytes.toBytes("2"));
772 t.put(p);
773 p = new Put(Bytes.toBytes("row3"));
774 p.add(CF, Bytes.toBytes("q1"), Bytes.toBytes("3"));
775 t.put(p);
776 p = new Put(Bytes.toBytes("row4"));
777 p.add(CF, Bytes.toBytes("q1"), Bytes.toBytes("4"));
778 t.put(p);
779 admin.flush(tableName);
780 }
781
782
783
784
785 @Test
786 public void testSplitRegionNotAssignable() throws Exception {
787 final TableName tableName =
788 TableName.valueOf("testSplitRegionWithNoStoreFiles");
789
790 HTable t = createTableAndWait(tableName.getName(), CF);
791 try {
792 List<HRegion> regions = cluster.getRegions(tableName);
793 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
794 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
795 insertData(tableName.getName(), admin, t);
796
797 admin.setBalancerRunning(false, true);
798
799 cluster.getMaster().setCatalogJanitorEnabled(false);
800 final HRegion region = findSplittableRegion(regions);
801 assertTrue("not able to find a splittable region", region != null);
802
803
804 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
805 try {
806 st.prepare();
807 st.execute(regionServer, regionServer);
808 } catch (IOException e) {
809 fail("Split execution should have succeeded with no exceptions thrown");
810 }
811
812 List<HRegion> daughters = cluster.getRegions(tableName);
813 assertTrue(daughters.size() == regions.size() + 1);
814
815 HRegionInfo hri = region.getRegionInfo();
816 AssignmentManager am = cluster.getMaster().getAssignmentManager();
817 RegionStates regionStates = am.getRegionStates();
818 long start = EnvironmentEdgeManager.currentTimeMillis();
819 while (!regionStates.isRegionInState(hri, State.SPLIT)) {
820 assertFalse("Timed out in waiting split parent to be in state SPLIT",
821 EnvironmentEdgeManager.currentTimeMillis() - start > 60000);
822 Thread.sleep(500);
823 }
824
825
826 am.assign(hri, true, true);
827 assertFalse("Split region can't be assigned",
828 regionStates.isRegionInTransition(hri));
829 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
830
831
832 am.unassign(hri, true, null);
833 assertFalse("Split region can't be unassigned",
834 regionStates.isRegionInTransition(hri));
835 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
836 } finally {
837 admin.setBalancerRunning(true, false);
838 cluster.getMaster().setCatalogJanitorEnabled(true);
839 }
840 }
841
842 private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
843 final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
844 try {
845
846 createTableAndWait(tableName, CF);
847
848 List<HRegion> regions = awaitTableRegions(tableName);
849 assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
850
851 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
852 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
853 final HRegion region = findSplittableRegion(regions);
854 assertTrue("not able to find a splittable region", region != null);
855 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
856 @Override
857 PairOfSameType<HRegion> createDaughters(final Server server,
858 final RegionServerServices services) throws IOException {
859 throw new SplittingNodeCreationFailedException ();
860 }
861 };
862 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
863 region.getRegionInfo().getEncodedName());
864 regionServer.getZooKeeper().sync(node);
865 for (int i = 0; i < 100; i++) {
866
867
868
869 if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
870 Thread.sleep(100);
871 }
872 }
873 try {
874 st.prepare();
875 st.execute(regionServer, regionServer);
876 } catch (IOException e) {
877
878
879
880 assertTrue("Should be instance of CreateSplittingNodeFailedException",
881 e instanceof SplittingNodeCreationFailedException );
882 node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
883 region.getRegionInfo().getEncodedName());
884 {
885 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
886 }
887 assertTrue(st.rollback(regionServer, regionServer));
888 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
889 }
890 } finally {
891 TESTING_UTIL.deleteTable(tableName);
892 }
893 }
894
895 public static class MockedSplitTransaction extends SplitTransaction {
896
897 private HRegion currentRegion;
898 public MockedSplitTransaction(HRegion r, byte[] splitrow) {
899 super(r, splitrow);
900 this.currentRegion = r;
901 }
902
903 @Override
904 void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b)
905 throws IOException {
906 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
907 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
908 try {
909 if (!secondSplit){
910 callRollBack = true;
911 latch.await();
912 }
913 } catch (InterruptedException e) {
914 }
915
916 }
917 super.transitionZKNode(server, services, a, b);
918 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
919 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
920 firstSplitCompleted = true;
921 }
922 }
923 @Override
924 public boolean rollback(Server server, RegionServerServices services) throws IOException {
925 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
926 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
927 if(secondSplit){
928 super.rollback(server, services);
929 latch.countDown();
930 return true;
931 }
932 }
933 return super.rollback(server, services);
934 }
935
936 }
937
938 private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
939 for (int i = 0; i < 5; ++i) {
940 for (HRegion r: regions) {
941 if (r.isSplittable()) {
942 return(r);
943 }
944 }
945 Thread.sleep(100);
946 }
947 return(null);
948 }
949
950 private List<HRegion> checkAndGetDaughters(byte[] tableName)
951 throws InterruptedException {
952 List<HRegion> daughters = null;
953
954 for (int i=0; i<100; i++) {
955 daughters = cluster.getRegions(tableName);
956 if (daughters.size() >= 2) break;
957 Thread.sleep(100);
958 }
959 assertTrue(daughters.size() >= 2);
960 return daughters;
961 }
962
963 private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
964 throws IOException, InterruptedException {
965 cluster.abortMaster(0);
966 cluster.waitOnMaster(0);
967 cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
968 MockMasterWithoutCatalogJanitor.class, HMaster.class);
969 MockMasterWithoutCatalogJanitor master = null;
970 master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
971 cluster.waitForActiveAndReadyMaster();
972 return master;
973 }
974
975 private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
976 throws IOException, InterruptedException {
977 this.admin.split(hri.getRegionNameAsString());
978 for (int i = 0; ProtobufUtil.getOnlineRegions(server).size() <= regionCount && i < 300; i++) {
979 LOG.debug("Waiting on region to split");
980 Thread.sleep(100);
981 }
982
983 assertFalse("Waited too long for split",
984 ProtobufUtil.getOnlineRegions(server).size() <= regionCount);
985 }
986
987
988
989
990
991
992
993
994
995
996
997
998 private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
999 final HRegionInfo hri)
1000 throws HBaseIOException, MasterNotRunningException,
1001 ZooKeeperConnectionException, InterruptedException {
1002
1003
1004
1005 int metaServerIndex = cluster.getServerWithMeta();
1006 assertTrue(metaServerIndex != -1);
1007 HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1008 int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1009 assertTrue(tableRegionIndex != -1);
1010 HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1011 if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1012 HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1013 assertNotNull(hrs);
1014 assertNotNull(hri);
1015 LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1016 metaRegionServer.getServerName() + " to " +
1017 hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1018 admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1019 }
1020
1021 for (int i = 0; i < 100; i++) {
1022 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1023 if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1024 LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1025 tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1026 Thread.sleep(100);
1027 }
1028 assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1029 && tableRegionIndex != metaServerIndex);
1030
1031 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1032 assertTrue(tableRegionIndex != -1);
1033 assertNotSame(metaServerIndex, tableRegionIndex);
1034 return tableRegionIndex;
1035 }
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046 private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1047 final HRegionServer notThisOne) {
1048 for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1049 HRegionServer hrs = rst.getRegionServer();
1050 if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1051 if (hrs.isStopping() || hrs.isStopped()) continue;
1052 return hrs;
1053 }
1054 return null;
1055 }
1056
1057 private void printOutRegions(final HRegionServer hrs, final String prefix)
1058 throws IOException {
1059 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
1060 for (HRegionInfo region: regions) {
1061 LOG.info(prefix + region.getRegionNameAsString());
1062 }
1063 }
1064
1065 private void waitUntilRegionServerDead() throws InterruptedException {
1066
1067 for (int i=0; cluster.getMaster().getClusterStatus().
1068 getServers().size() == NB_SERVERS && i<100; i++) {
1069 LOG.info("Waiting on server to go down");
1070 Thread.sleep(100);
1071 }
1072 assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1073 getServers().size() == NB_SERVERS);
1074 }
1075
1076 private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
1077
1078 for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1079 LOG.info("Waiting for repair to happen");
1080 Thread.sleep(1000);
1081 }
1082 if (cluster.getRegions(tableName).size() < numDaughters) {
1083 fail("Waiting too long for daughter regions");
1084 }
1085 }
1086
1087 private List<HRegion> awaitTableRegions(final byte[] tableName) throws InterruptedException {
1088 List<HRegion> regions = null;
1089 for (int i = 0; i < 100; i++) {
1090 regions = cluster.getRegions(tableName);
1091 if (regions.size() > 0) break;
1092 Thread.sleep(100);
1093 }
1094 return regions;
1095 }
1096
1097 private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException,
1098 InterruptedException {
1099 HTable t = TESTING_UTIL.createTable(tableName, cf);
1100 awaitTableRegions(tableName);
1101 assertTrue("Table not online: " + Bytes.toString(tableName),
1102 cluster.getRegions(tableName).size() != 0);
1103 return t;
1104 }
1105
1106 public static class MockMasterWithoutCatalogJanitor extends HMaster {
1107
1108 public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException,
1109 InterruptedException {
1110 super(conf);
1111 }
1112
1113 protected void startCatalogJanitorChore() {
1114 LOG.debug("Customised master executed.");
1115 }
1116 }
1117
1118 private static class SplittingNodeCreationFailedException extends IOException {
1119 private static final long serialVersionUID = 1652404976265623004L;
1120
1121 public SplittingNodeCreationFailedException () {
1122 super();
1123 }
1124 }
1125 }
1126