1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNotSame;
25 import static org.junit.Assert.assertNull;
26 import static org.junit.Assert.assertTrue;
27 import static org.junit.Assert.fail;
28
29 import java.io.IOException;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.concurrent.CountDownLatch;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.Abortable;
40 import org.apache.hadoop.hbase.Coprocessor;
41 import org.apache.hadoop.hbase.HBaseIOException;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.LargeTests;
48 import org.apache.hadoop.hbase.MasterNotRunningException;
49 import org.apache.hadoop.hbase.MiniHBaseCluster;
50 import org.apache.hadoop.hbase.RegionTransition;
51 import org.apache.hadoop.hbase.Server;
52 import org.apache.hadoop.hbase.ServerName;
53 import org.apache.hadoop.hbase.TableName;
54 import org.apache.hadoop.hbase.UnknownRegionException;
55 import org.apache.hadoop.hbase.Waiter;
56 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
57 import org.apache.hadoop.hbase.catalog.MetaEditor;
58 import org.apache.hadoop.hbase.catalog.MetaReader;
59 import org.apache.hadoop.hbase.client.Delete;
60 import org.apache.hadoop.hbase.client.HBaseAdmin;
61 import org.apache.hadoop.hbase.client.HTable;
62 import org.apache.hadoop.hbase.client.Mutation;
63 import org.apache.hadoop.hbase.client.Put;
64 import org.apache.hadoop.hbase.client.Result;
65 import org.apache.hadoop.hbase.client.ResultScanner;
66 import org.apache.hadoop.hbase.client.Scan;
67 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
68 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
69 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
70 import org.apache.hadoop.hbase.exceptions.DeserializationException;
71 import org.apache.hadoop.hbase.executor.EventType;
72 import org.apache.hadoop.hbase.master.AssignmentManager;
73 import org.apache.hadoop.hbase.master.HMaster;
74 import org.apache.hadoop.hbase.master.RegionState;
75 import org.apache.hadoop.hbase.master.RegionStates;
76 import org.apache.hadoop.hbase.master.RegionState.State;
77 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
78 import org.apache.hadoop.hbase.util.Bytes;
79 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
80 import org.apache.hadoop.hbase.util.FSUtils;
81 import org.apache.hadoop.hbase.util.HBaseFsck;
82 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
83 import org.apache.hadoop.hbase.util.PairOfSameType;
84 import org.apache.hadoop.hbase.util.Threads;
85 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
86 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
87 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
88 import org.apache.zookeeper.KeeperException;
89 import org.apache.zookeeper.KeeperException.NodeExistsException;
90 import org.apache.zookeeper.data.Stat;
91 import org.junit.After;
92 import org.junit.AfterClass;
93 import org.junit.Assert;
94 import org.junit.Before;
95 import org.junit.BeforeClass;
96 import org.junit.Test;
97 import org.junit.experimental.categories.Category;
98
99 import com.google.protobuf.ServiceException;
100
101
102
103
104
105
106 @Category(LargeTests.class)
107 public class TestSplitTransactionOnCluster {
108 private static final Log LOG =
109 LogFactory.getLog(TestSplitTransactionOnCluster.class);
110 private HBaseAdmin admin = null;
111 private MiniHBaseCluster cluster = null;
112 private static final int NB_SERVERS = 3;
113 private static CountDownLatch latch = new CountDownLatch(1);
114 private static volatile boolean secondSplit = false;
115 private static volatile boolean callRollBack = false;
116 private static volatile boolean firstSplitCompleted = false;
117 private static boolean useZKForAssignment = true;
118
119 static final HBaseTestingUtility TESTING_UTIL =
120 new HBaseTestingUtility();
121
122 static void setupOnce() throws Exception {
123 TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
124 useZKForAssignment =
125 TESTING_UTIL.getConfiguration().getBoolean("hbase.assignment.usezk", false);
126 TESTING_UTIL.startMiniCluster(NB_SERVERS);
127 }
128
129 @BeforeClass public static void before() throws Exception {
130
131 TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
132 setupOnce();
133 }
134
135 @AfterClass public static void after() throws Exception {
136 TESTING_UTIL.shutdownMiniCluster();
137 }
138
139 @Before public void setup() throws IOException {
140 TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
141 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
142 this.cluster = TESTING_UTIL.getMiniHBaseCluster();
143 }
144
145 @After
146 public void tearDown() throws Exception {
147 this.admin.close();
148 }
149
150 private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
151 assertEquals(1, regions.size());
152 HRegionInfo hri = regions.get(0).getRegionInfo();
153 return waitOnRIT(hri);
154 }
155
156
157
158
159
160
161
162
163 private HRegionInfo waitOnRIT(final HRegionInfo hri) {
164
165
166 while (TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
167 getRegionStates().isRegionInTransition(hri)) {
168 LOG.info("Waiting on region in transition: " +
169 TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().
170 getRegionTransitionState(hri));
171 Threads.sleep(10);
172 }
173 return hri;
174 }
175
176 @SuppressWarnings("deprecation")
177 @Test(timeout = 60000)
178 public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
179 final TableName tableName =
180 TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
181
182 if (!useZKForAssignment) {
183
184 return;
185 }
186
187 try {
188
189 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
190 final List<HRegion> regions = cluster.getRegions(tableName);
191 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
192 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
193 final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
194 insertData(tableName.getName(), admin, t);
195 t.close();
196
197
198 this.admin.setBalancerRunning(false, true);
199
200 cluster.getMaster().setCatalogJanitorEnabled(false);
201
202
203 final HRegion region = findSplittableRegion(regions);
204 assertTrue("not able to find a splittable region", region != null);
205
206 new Thread() {
207 @Override
208 public void run() {
209 SplitTransaction st = null;
210 st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
211 try {
212 st.prepare();
213 st.execute(regionServer, regionServer);
214 } catch (IOException e) {
215
216 }
217 }
218 }.start();
219 for (int i = 0; !callRollBack && i < 100; i++) {
220 Thread.sleep(100);
221 }
222 assertTrue("Waited too long for rollback", callRollBack);
223 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
224 try {
225 secondSplit = true;
226
227 region.initialize();
228 st.prepare();
229 st.execute(regionServer, regionServer);
230 } catch (IOException e) {
231 LOG.debug("Rollback started :"+ e.getMessage());
232 st.rollback(regionServer, regionServer);
233 }
234 for (int i=0; !firstSplitCompleted && i<100; i++) {
235 Thread.sleep(100);
236 }
237 assertTrue("fist split did not complete", firstSplitCompleted);
238
239 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
240 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
241
242 for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
243 Thread.sleep(100);
244 }
245 assertFalse("region still in transition", rit.containsKey(
246 rit.containsKey(hri.getTable())));
247
248 List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
249
250 assertEquals("The parent region should be splitted", 2, onlineRegions.size());
251
252 List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
253 .getRegionStates().getRegionsOfTable(tableName);
254
255 assertEquals("No of regions in master", 2, regionsOfTable.size());
256 } finally {
257 admin.setBalancerRunning(true, false);
258 secondSplit = false;
259 firstSplitCompleted = false;
260 callRollBack = false;
261 cluster.getMaster().setCatalogJanitorEnabled(true);
262 TESTING_UTIL.deleteTable(tableName);
263 }
264 }
265
266 @Test(timeout = 60000)
267 public void testRITStateForRollback() throws Exception {
268 final TableName tableName =
269 TableName.valueOf("testRITStateForRollback");
270 try {
271
272 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
273 final List<HRegion> regions = cluster.getRegions(tableName);
274 final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
275 insertData(tableName.getName(), admin, t);
276 t.close();
277
278
279 this.admin.setBalancerRunning(false, true);
280
281 cluster.getMaster().setCatalogJanitorEnabled(false);
282
283
284 final HRegion region = findSplittableRegion(regions);
285 assertTrue("not able to find a splittable region", region != null);
286
287
288 region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
289 Coprocessor.PRIORITY_USER, region.getBaseConf());
290
291
292 this.admin.split(region.getRegionName(), new byte[] {42});
293
294
295 FailingSplitRegionObserver.latch.await();
296
297 LOG.info("Waiting for region to come out of RIT");
298 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
299 @Override
300 public boolean evaluate() throws Exception {
301 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
302 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
303 return !rit.containsKey(hri.getEncodedName());
304 }
305 });
306 } finally {
307 admin.setBalancerRunning(true, false);
308 cluster.getMaster().setCatalogJanitorEnabled(true);
309 TESTING_UTIL.deleteTable(tableName);
310 }
311 }
312
313 public static class FailingSplitRegionObserver extends BaseRegionObserver {
314 static volatile CountDownLatch latch = new CountDownLatch(1);
315 @Override
316 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
317 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
318 latch.countDown();
319 throw new IOException("Causing rollback of region split");
320 }
321 }
322
323
324
325
326
327
328
329
330
331
332
333
334 @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
335 throws IOException, InterruptedException, NodeExistsException, KeeperException,
336 DeserializationException, ServiceException {
337 final byte [] tableName =
338 Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
339
340
341 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
342 List<HRegion> regions = cluster.getRegions(tableName);
343 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
344
345 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
346
347
348 this.admin.setBalancerRunning(false, true);
349
350 cluster.getMaster().setCatalogJanitorEnabled(false);
351 try {
352
353 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
354
355 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
356 printOutRegions(server, "Initial regions: ");
357 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
358
359
360 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
361
362 split(hri, server, regionCount);
363
364 String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
365 hri.getEncodedName());
366 RegionTransition rt = null;
367 Stat stats = null;
368 List<HRegion> daughters = null;
369 if (useZKForAssignment) {
370 daughters = checkAndGetDaughters(tableName);
371
372
373 for (int i=0; i<100; i++) {
374 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
375 rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
376 hri.getEncodedName()));
377 if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
378 Thread.sleep(100);
379 }
380 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
381 assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
382
383 cluster.abortRegionServer(tableRegionIndex);
384 }
385 waitUntilRegionServerDead();
386 awaitDaughters(tableName, 2);
387 if (useZKForAssignment) {
388 regions = cluster.getRegions(tableName);
389 for (HRegion r: regions) {
390 assertTrue(daughters.contains(r));
391 }
392
393
394 for (int i=0; i<100; i++) {
395
396 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
397 if (stats == null) break;
398 Thread.sleep(100);
399 }
400 LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
401 assertTrue(stats == null);
402 }
403 } finally {
404
405 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
406 admin.setBalancerRunning(true, false);
407 cluster.getMaster().setCatalogJanitorEnabled(true);
408 cluster.startRegionServer();
409 t.close();
410 }
411 }
412
413 @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
414 throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
415 final byte [] tableName =
416 Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
417
418
419 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
420 List<HRegion> regions = cluster.getRegions(tableName);
421 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
422
423 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
424
425 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
426
427
428 this.admin.setBalancerRunning(false, true);
429
430 cluster.getMaster().setCatalogJanitorEnabled(false);
431 try {
432
433 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
434
435 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
436 printOutRegions(server, "Initial regions: ");
437 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
438
439
440 ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
441 if (useZKForAssignment) {
442 ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
443 hri, fakedServer);
444 } else {
445 regionStates.updateRegionState(hri, RegionState.State.CLOSING);
446 }
447
448
449 this.admin.split(hri.getRegionNameAsString());
450 this.admin.split(hri.getRegionNameAsString());
451 this.admin.split(hri.getRegionNameAsString());
452
453 for (int i = 0; i < 10; i++) {
454 Thread.sleep(100);
455 assertEquals(regionCount, ProtobufUtil.getOnlineRegions(server).size());
456 }
457 if (useZKForAssignment) {
458
459 ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
460 hri, fakedServer);
461 } else {
462 regionStates.regionOnline(hri, server.getServerName());
463 }
464
465 split(hri, server, regionCount);
466
467 checkAndGetDaughters(tableName);
468
469 } finally {
470 admin.setBalancerRunning(true, false);
471 cluster.getMaster().setCatalogJanitorEnabled(true);
472 t.close();
473 }
474 }
475
476
477
478
479
480
481
482 @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
483 throws IOException, InterruptedException, ServiceException {
484 final byte [] tableName =
485 Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
486
487
488 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
489 List<HRegion> regions = cluster.getRegions(tableName);
490 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
491
492 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
493
494
495 this.admin.setBalancerRunning(false, true);
496
497 cluster.getMaster().setCatalogJanitorEnabled(false);
498 try {
499
500 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
501
502 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
503 printOutRegions(server, "Initial regions: ");
504 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
505
506 split(hri, server, regionCount);
507
508 List<HRegion> daughters = checkAndGetDaughters(tableName);
509
510 regionCount = ProtobufUtil.getOnlineRegions(server).size();
511 HRegionInfo daughter = daughters.get(0).getRegionInfo();
512 LOG.info("Daughter we are going to split: " + daughter);
513
514
515 this.admin.compact(daughter.getRegionName());
516 daughters = cluster.getRegions(tableName);
517 HRegion daughterRegion = null;
518 for (HRegion r: daughters) {
519 if (r.getRegionInfo().equals(daughter)) {
520 daughterRegion = r;
521 LOG.info("Found matching HRI: " + daughterRegion);
522 break;
523 }
524 }
525 assertTrue(daughterRegion != null);
526 for (int i=0; i<100; i++) {
527 if (!daughterRegion.hasReferences()) break;
528 Threads.sleep(100);
529 }
530 assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
531 LOG.info("Daughter hri before split (has been compacted): " + daughter);
532 split(daughter, server, regionCount);
533
534 daughters = cluster.getRegions(tableName);
535 for (HRegion d: daughters) {
536 LOG.info("Regions before crash: " + d);
537 }
538
539 cluster.abortRegionServer(tableRegionIndex);
540 waitUntilRegionServerDead();
541 awaitDaughters(tableName, daughters.size());
542
543
544 regions = cluster.getRegions(tableName);
545 for (HRegion d: daughters) {
546 LOG.info("Regions after crash: " + d);
547 }
548 assertEquals(daughters.size(), regions.size());
549 for (HRegion r: regions) {
550 LOG.info("Regions post crash " + r);
551 assertTrue("Missing region post crash " + r, daughters.contains(r));
552 }
553 } finally {
554 admin.setBalancerRunning(true, false);
555 cluster.getMaster().setCatalogJanitorEnabled(true);
556 t.close();
557 }
558 }
559
560 @Test(timeout = 180000)
561 public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
562 Configuration conf = TESTING_UTIL.getConfiguration();
563 TableName userTableName =
564 TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
565 HTableDescriptor htd = new HTableDescriptor(userTableName);
566 HColumnDescriptor hcd = new HColumnDescriptor("col");
567 htd.addFamily(hcd);
568 admin.createTable(htd);
569 HTable table = new HTable(conf, userTableName);
570 try {
571 for (int i = 0; i <= 5; i++) {
572 String row = "row" + i;
573 Put p = new Put(row.getBytes());
574 String val = "Val" + i;
575 p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
576 table.put(p);
577 admin.flush(userTableName.getName());
578 Delete d = new Delete(row.getBytes());
579
580 table.delete(d);
581 admin.flush(userTableName.getName());
582 }
583 admin.majorCompact(userTableName.getName());
584 List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
585 .getMaster().getAssignmentManager().getRegionStates()
586 .getRegionsOfTable(userTableName);
587 HRegionInfo hRegionInfo = regionsOfTable.get(0);
588 Put p = new Put("row6".getBytes());
589 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
590 table.put(p);
591 p = new Put("row7".getBytes());
592 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
593 table.put(p);
594 p = new Put("row8".getBytes());
595 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
596 table.put(p);
597 admin.flush(userTableName.getName());
598 admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
599 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
600 .getAssignmentManager().getRegionStates()
601 .getRegionsOfTable(userTableName);
602
603 while (regionsOfTable.size() != 2) {
604 Thread.sleep(2000);
605 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
606 .getAssignmentManager().getRegionStates()
607 .getRegionsOfTable(userTableName);
608 }
609 Assert.assertEquals(2, regionsOfTable.size());
610 Scan s = new Scan();
611 ResultScanner scanner = table.getScanner(s);
612 int mainTableCount = 0;
613 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
614 mainTableCount++;
615 }
616 Assert.assertEquals(3, mainTableCount);
617 } finally {
618 table.close();
619 }
620 }
621
622
623
624
625 static class UselessTestAbortable implements Abortable {
626 boolean aborted = false;
627 @Override
628 public void abort(String why, Throwable e) {
629 LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
630 aborted = true;
631 }
632
633 @Override
634 public boolean isAborted() {
635 return this.aborted;
636 }
637 }
638
639
640
641
642
643
644
645
646
647
648
649 @Test(timeout = 400000)
650 public void testMasterRestartWhenSplittingIsPartial()
651 throws IOException, InterruptedException, NodeExistsException,
652 KeeperException, DeserializationException, ServiceException {
653 final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
654
655 if (!useZKForAssignment) {
656
657 return;
658 }
659
660
661 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
662 List<HRegion> regions = cluster.getRegions(tableName);
663 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
664
665 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
666
667
668 this.admin.setBalancerRunning(false, true);
669
670 cluster.getMaster().setCatalogJanitorEnabled(false);
671 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
672 "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
673 try {
674
675 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
676
677 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
678 printOutRegions(server, "Initial regions: ");
679
680
681 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
682
683
684 this.admin.split(hri.getRegionNameAsString());
685 checkAndGetDaughters(tableName);
686
687 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
688 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
689 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
690 + stats);
691 byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
692 RegionTransition rtd = RegionTransition.parseFrom(bytes);
693
694 assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
695 || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
696
697
698 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
699
700 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
701
702
703
704 hri.setOffline(true);
705 hri.setSplit(true);
706 ServerName regionServerOfRegion = master.getAssignmentManager()
707 .getRegionStates().getRegionServerOfRegion(hri);
708 assertTrue(regionServerOfRegion != null);
709
710
711 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
712 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
713 Stat stat = new Stat();
714 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
715
716 for (int i=0; data != null && i<60; i++) {
717 Thread.sleep(1000);
718 data = ZKUtil.getDataNoWatch(zkw, node, stat);
719 }
720 assertNull("Waited too long for ZK node to be removed: "+node, data);
721 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
722 assertTrue("Split parent should be in SPLIT state",
723 regionStates.isRegionInState(hri, State.SPLIT));
724 regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
725 assertTrue(regionServerOfRegion == null);
726 } finally {
727
728 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
729 admin.setBalancerRunning(true, false);
730 cluster.getMaster().setCatalogJanitorEnabled(true);
731 t.close();
732 zkw.close();
733 }
734 }
735
736
737
738
739
740
741
742
743
744 @Test (timeout = 300000)
745 public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
746 throws IOException, InterruptedException, NodeExistsException,
747 KeeperException, ServiceException {
748 final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
749
750
751 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
752 List<HRegion> regions = cluster.getRegions(tableName);
753 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
754
755 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
756
757
758 this.admin.setBalancerRunning(false, true);
759
760 cluster.getMaster().setCatalogJanitorEnabled(false);
761 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
762 "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
763 try {
764
765 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
766
767 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
768 printOutRegions(server, "Initial regions: ");
769
770 this.admin.split(hri.getRegionNameAsString());
771 checkAndGetDaughters(tableName);
772
773 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
774 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
775 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
776 + stats);
777 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
778 Stat stat = new Stat();
779 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
780
781 for (int i=0; data != null && i<60; i++) {
782 Thread.sleep(1000);
783 data = ZKUtil.getDataNoWatch(zkw, node, stat);
784 }
785 assertNull("Waited too long for ZK node to be removed: "+node, data);
786
787 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
788
789 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
790
791
792
793 hri.setOffline(true);
794 hri.setSplit(true);
795 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
796 assertTrue("Split parent should be in SPLIT state",
797 regionStates.isRegionInState(hri, State.SPLIT));
798 ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
799 assertTrue(regionServerOfRegion == null);
800 } finally {
801 this.admin.setBalancerRunning(true, false);
802 cluster.getMaster().setCatalogJanitorEnabled(true);
803 t.close();
804 zkw.close();
805 }
806 }
807
808
809
810
811
812
813
814
815
816
817
818
819 @Test(timeout = 60000)
820 public void testSplitBeforeSettingSplittingInZK() throws Exception,
821 InterruptedException, KeeperException {
822 testSplitBeforeSettingSplittingInZKInternals();
823 }
824
825 @Test(timeout = 60000)
826 public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
827 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
828 final TableName tableName =
829 TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
830
831 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
832 List<HRegion> regions = null;
833 try {
834 regions = cluster.getRegions(tableName);
835 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
836 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
837 insertData(tableName.getName(), admin, t);
838
839 admin.setBalancerRunning(false, true);
840
841 cluster.getMaster().setCatalogJanitorEnabled(false);
842 boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
843 tableName);
844 assertEquals("The specified table should present.", true, tableExists);
845 final HRegion region = findSplittableRegion(regions);
846 assertTrue("not able to find a splittable region", region != null);
847 SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
848 try {
849 st.prepare();
850 st.createDaughters(regionServer, regionServer);
851 } catch (IOException e) {
852
853 }
854 tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
855 tableName);
856 assertEquals("The specified table should present.", true, tableExists);
857 } finally {
858 if (regions != null) {
859 String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
860 .getEncodedName());
861 ZKUtil.deleteNodeFailSilent(zkw, node);
862 }
863 admin.setBalancerRunning(true, false);
864 cluster.getMaster().setCatalogJanitorEnabled(true);
865 t.close();
866 }
867 }
868
869 private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
870 InterruptedException {
871 Put p = new Put(Bytes.toBytes("row1"));
872 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
873 t.put(p);
874 p = new Put(Bytes.toBytes("row2"));
875 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
876 t.put(p);
877 p = new Put(Bytes.toBytes("row3"));
878 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
879 t.put(p);
880 p = new Put(Bytes.toBytes("row4"));
881 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
882 t.put(p);
883 admin.flush(tableName);
884 }
885
886
887
888
889
890 @Test(timeout = 60000)
891 public void testSplitRegionWithNoStoreFiles()
892 throws Exception {
893 final TableName tableName =
894 TableName.valueOf("testSplitRegionWithNoStoreFiles");
895
896 createTableAndWait(tableName.getName(), HConstants.CATALOG_FAMILY);
897 List<HRegion> regions = cluster.getRegions(tableName);
898 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
899 ensureTableRegionNotOnSameServerAsMeta(admin, hri);
900 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
901 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
902
903 this.admin.setBalancerRunning(false, true);
904
905 cluster.getMaster().setCatalogJanitorEnabled(false);
906 try {
907
908 printOutRegions(regionServer, "Initial regions: ");
909 Configuration conf = cluster.getConfiguration();
910 HBaseFsck.debugLsr(conf, new Path("/"));
911 Path rootDir = FSUtils.getRootDir(conf);
912 FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
913 Map<String, Path> storefiles =
914 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
915 assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
916
917
918 regions = cluster.getRegions(tableName);
919 final HRegion region = findSplittableRegion(regions);
920 assertTrue("not able to find a splittable region", region != null);
921
922
923 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
924 try {
925 st.prepare();
926 st.execute(regionServer, regionServer);
927 } catch (IOException e) {
928 fail("Split execution should have succeeded with no exceptions thrown");
929 }
930
931
932
933 List<HRegion> daughters = cluster.getRegions(tableName);
934 assertTrue(daughters.size() == 2);
935
936
937 HBaseFsck.debugLsr(conf, new Path("/"));
938 Map<String, Path> storefilesAfter =
939 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
940 assertEquals("Expected nothing but found " + storefilesAfter.toString(),
941 storefilesAfter.size(), 0);
942
943 hri = region.getRegionInfo();
944 AssignmentManager am = cluster.getMaster().getAssignmentManager();
945 RegionStates regionStates = am.getRegionStates();
946 long start = EnvironmentEdgeManager.currentTimeMillis();
947 while (!regionStates.isRegionInState(hri, State.SPLIT)) {
948 assertFalse("Timed out in waiting split parent to be in state SPLIT",
949 EnvironmentEdgeManager.currentTimeMillis() - start > 60000);
950 Thread.sleep(500);
951 }
952
953
954 am.assign(hri, true, true);
955 assertFalse("Split region can't be assigned",
956 regionStates.isRegionInTransition(hri));
957 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
958
959
960 am.unassign(hri, true, null);
961 assertFalse("Split region can't be unassigned",
962 regionStates.isRegionInTransition(hri));
963 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
964 } finally {
965 admin.setBalancerRunning(true, false);
966 cluster.getMaster().setCatalogJanitorEnabled(true);
967 }
968 }
969
970 @Test(timeout = 180000)
971 public void testSplitHooksBeforeAndAfterPONR() throws Exception {
972 String firstTable = "testSplitHooksBeforeAndAfterPONR_1";
973 String secondTable = "testSplitHooksBeforeAndAfterPONR_2";
974 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(firstTable));
975 desc.addCoprocessor(MockedRegionObserver.class.getName());
976 HColumnDescriptor hcd = new HColumnDescriptor("cf");
977 desc.addFamily(hcd);
978 admin.createTable(desc);
979 desc = new HTableDescriptor(TableName.valueOf(secondTable));
980 hcd = new HColumnDescriptor("cf");
981 desc.addFamily(hcd);
982 admin.createTable(desc);
983 List<HRegion> firstTableregions = cluster.getRegions(TableName.valueOf(firstTable));
984 List<HRegion> secondTableRegions = cluster.getRegions(TableName.valueOf(secondTable));
985 ServerName serverName =
986 cluster.getServerHoldingRegion(firstTableregions.get(0).getRegionName());
987 admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
988 Bytes.toBytes(serverName.getServerName()));
989 HTable table1 = null;
990 HTable table2 = null;
991 try {
992 table1 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
993 table2 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
994 insertData(Bytes.toBytes(firstTable), admin, table1);
995 insertData(Bytes.toBytes(secondTable), admin, table2);
996 admin.split(Bytes.toBytes(firstTable), "row2".getBytes());
997 firstTableregions = cluster.getRegions(Bytes.toBytes(firstTable));
998 while (firstTableregions.size() != 2) {
999 Thread.sleep(1000);
1000 firstTableregions = cluster.getRegions(Bytes.toBytes(firstTable));
1001 }
1002 assertEquals("Number of regions after split should be 2.", 2, firstTableregions.size());
1003 secondTableRegions = cluster.getRegions(Bytes.toBytes(secondTable));
1004 assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
1005 } finally {
1006 if (table1 != null) {
1007 table1.close();
1008 }
1009 if (table2 != null) {
1010 table2.close();
1011 }
1012 TESTING_UTIL.deleteTable(firstTable);
1013 TESTING_UTIL.deleteTable(secondTable);
1014 }
1015 }
1016
1017 private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
1018 final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
1019 try {
1020
1021 createTableAndWait(tableName, Bytes.toBytes("cf"));
1022
1023 List<HRegion> regions = awaitTableRegions(tableName);
1024 assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
1025
1026 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
1027 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1028 final HRegion region = findSplittableRegion(regions);
1029 assertTrue("not able to find a splittable region", region != null);
1030 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
1031 @Override
1032 public PairOfSameType<HRegion> stepsBeforePONR(final Server server,
1033 final RegionServerServices services, boolean testing) throws IOException {
1034 throw new SplittingNodeCreationFailedException ();
1035 }
1036 };
1037 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1038 region.getRegionInfo().getEncodedName());
1039 regionServer.getZooKeeper().sync(node);
1040 for (int i = 0; i < 100; i++) {
1041
1042
1043
1044 if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
1045 Thread.sleep(100);
1046 }
1047 }
1048 try {
1049 st.prepare();
1050 st.execute(regionServer, regionServer);
1051 } catch (IOException e) {
1052
1053
1054
1055 assertTrue("Should be instance of CreateSplittingNodeFailedException",
1056 e instanceof SplittingNodeCreationFailedException );
1057 node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1058 region.getRegionInfo().getEncodedName());
1059 {
1060 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1061 }
1062 assertTrue(st.rollback(regionServer, regionServer));
1063 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1064 }
1065 } finally {
1066 TESTING_UTIL.deleteTable(tableName);
1067 }
1068 }
1069
1070 public static class MockedSplitTransaction extends SplitTransaction {
1071
1072 private HRegion currentRegion;
1073 public MockedSplitTransaction(HRegion r, byte[] splitrow) {
1074 super(r, splitrow);
1075 this.currentRegion = r;
1076 }
1077
1078 @Override
1079 void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b)
1080 throws IOException {
1081 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1082 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1083 try {
1084 if (!secondSplit){
1085 callRollBack = true;
1086 latch.await();
1087 }
1088 } catch (InterruptedException e) {
1089 }
1090
1091 }
1092 super.transitionZKNode(server, services, a, b);
1093 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1094 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1095 firstSplitCompleted = true;
1096 }
1097 }
1098 @Override
1099 public boolean rollback(Server server, RegionServerServices services) throws IOException {
1100 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1101 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1102 if(secondSplit){
1103 super.rollback(server, services);
1104 latch.countDown();
1105 return true;
1106 }
1107 }
1108 return super.rollback(server, services);
1109 }
1110
1111 }
1112
1113 private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
1114 for (int i = 0; i < 5; ++i) {
1115 for (HRegion r: regions) {
1116 if (r.isSplittable()) {
1117 return(r);
1118 }
1119 }
1120 Thread.sleep(100);
1121 }
1122 return(null);
1123 }
1124
1125 private List<HRegion> checkAndGetDaughters(byte[] tableName)
1126 throws InterruptedException {
1127 List<HRegion> daughters = null;
1128
1129 for (int i=0; i<100; i++) {
1130 daughters = cluster.getRegions(tableName);
1131 if (daughters.size() >= 2) break;
1132 Thread.sleep(100);
1133 }
1134 assertTrue(daughters.size() >= 2);
1135 return daughters;
1136 }
1137
1138 private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
1139 throws IOException, InterruptedException {
1140 cluster.abortMaster(0);
1141 cluster.waitOnMaster(0);
1142 cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
1143 MockMasterWithoutCatalogJanitor.class, HMaster.class);
1144 MockMasterWithoutCatalogJanitor master = null;
1145 master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
1146 cluster.waitForActiveAndReadyMaster();
1147 return master;
1148 }
1149
1150 private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
1151 throws IOException, InterruptedException {
1152 this.admin.split(hri.getRegionNameAsString());
1153 try {
1154 for (int i = 0; ProtobufUtil.getOnlineRegions(server).size() <= regionCount && i < 300; i++) {
1155 LOG.debug("Waiting on region to split");
1156 Thread.sleep(100);
1157 }
1158
1159 assertFalse("Waited too long for split",
1160 ProtobufUtil.getOnlineRegions(server).size() <= regionCount);
1161 } catch (RegionServerStoppedException e) {
1162 if (useZKForAssignment) {
1163
1164 LOG.error(e);
1165 throw e;
1166 }
1167 }
1168 }
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181 private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
1182 final HRegionInfo hri)
1183 throws HBaseIOException, MasterNotRunningException,
1184 ZooKeeperConnectionException, InterruptedException {
1185
1186
1187
1188 int metaServerIndex = cluster.getServerWithMeta();
1189 assertTrue(metaServerIndex != -1);
1190 HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1191 int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1192 assertTrue(tableRegionIndex != -1);
1193 HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1194 if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1195 HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1196 assertNotNull(hrs);
1197 assertNotNull(hri);
1198 LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1199 metaRegionServer.getServerName() + " to " +
1200 hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1201 admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1202 }
1203
1204 for (int i = 0; i < 100; i++) {
1205 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1206 if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1207 LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1208 tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1209 Thread.sleep(100);
1210 }
1211 assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1212 && tableRegionIndex != metaServerIndex);
1213
1214 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1215 assertTrue(tableRegionIndex != -1);
1216 assertNotSame(metaServerIndex, tableRegionIndex);
1217 return tableRegionIndex;
1218 }
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229 private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1230 final HRegionServer notThisOne) {
1231 for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1232 HRegionServer hrs = rst.getRegionServer();
1233 if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1234 if (hrs.isStopping() || hrs.isStopped()) continue;
1235 return hrs;
1236 }
1237 return null;
1238 }
1239
1240 private void printOutRegions(final HRegionServer hrs, final String prefix)
1241 throws IOException {
1242 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
1243 for (HRegionInfo region: regions) {
1244 LOG.info(prefix + region.getRegionNameAsString());
1245 }
1246 }
1247
1248 private void waitUntilRegionServerDead() throws InterruptedException {
1249
1250 for (int i=0; cluster.getMaster().getClusterStatus().
1251 getServers().size() == NB_SERVERS && i<100; i++) {
1252 LOG.info("Waiting on server to go down");
1253 Thread.sleep(100);
1254 }
1255 assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1256 getServers().size() == NB_SERVERS);
1257 }
1258
1259 private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
1260
1261 for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1262 LOG.info("Waiting for repair to happen");
1263 Thread.sleep(1000);
1264 }
1265 if (cluster.getRegions(tableName).size() < numDaughters) {
1266 fail("Waiting too long for daughter regions");
1267 }
1268 }
1269
1270 private List<HRegion> awaitTableRegions(final byte[] tableName) throws InterruptedException {
1271 List<HRegion> regions = null;
1272 for (int i = 0; i < 100; i++) {
1273 regions = cluster.getRegions(tableName);
1274 if (regions.size() > 0) break;
1275 Thread.sleep(100);
1276 }
1277 return regions;
1278 }
1279
1280 private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException,
1281 InterruptedException {
1282 HTable t = TESTING_UTIL.createTable(tableName, cf);
1283 awaitTableRegions(tableName);
1284 assertTrue("Table not online: " + Bytes.toString(tableName),
1285 cluster.getRegions(tableName).size() != 0);
1286 return t;
1287 }
1288
1289 public static class MockMasterWithoutCatalogJanitor extends HMaster {
1290
1291 public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException,
1292 InterruptedException {
1293 super(conf);
1294 }
1295 }
1296
1297 private static class SplittingNodeCreationFailedException extends IOException {
1298 private static final long serialVersionUID = 1652404976265623004L;
1299
1300 public SplittingNodeCreationFailedException () {
1301 super();
1302 }
1303 }
1304
1305 public static class MockedRegionObserver extends BaseRegionObserver {
1306 private SplitTransaction st = null;
1307 private PairOfSameType<HRegion> daughterRegions = null;
1308
1309 @Override
1310 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
1311 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
1312 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1313 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1314 List<HRegion> onlineRegions =
1315 rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
1316 HRegion region = onlineRegions.get(0);
1317 for (HRegion r : onlineRegions) {
1318 if (r.getRegionInfo().containsRow(splitKey)) {
1319 region = r;
1320 break;
1321 }
1322 }
1323 st = new SplitTransaction(region, splitKey);
1324 if (!st.prepare()) {
1325 LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
1326 + " failed. So returning null. ");
1327 ctx.bypass();
1328 return;
1329 }
1330 region.forceSplit(splitKey);
1331 daughterRegions = st.stepsBeforePONR(rs, rs, false);
1332 HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
1333 copyOfParent.setOffline(true);
1334 copyOfParent.setSplit(true);
1335
1336 Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
1337 MetaEditor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
1338 daughterRegions.getSecond().getRegionInfo());
1339 metaEntries.add(putParent);
1340
1341 Put putA = MetaEditor.makePutFromRegionInfo(daughterRegions.getFirst().getRegionInfo());
1342 Put putB = MetaEditor.makePutFromRegionInfo(daughterRegions.getSecond().getRegionInfo());
1343 st.addLocation(putA, rs.getServerName(), 1);
1344 st.addLocation(putB, rs.getServerName(), 1);
1345 metaEntries.add(putA);
1346 metaEntries.add(putB);
1347 }
1348
1349 @Override
1350 public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
1351 throws IOException {
1352 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1353 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1354 st.stepsAfterPONR(rs, rs, daughterRegions);
1355 }
1356
1357 }
1358 }
1359