1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNotSame;
25 import static org.junit.Assert.assertNull;
26 import static org.junit.Assert.assertTrue;
27 import static org.junit.Assert.fail;
28
29 import java.io.IOException;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.concurrent.CountDownLatch;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.Abortable;
40 import org.apache.hadoop.hbase.Coprocessor;
41 import org.apache.hadoop.hbase.HBaseIOException;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.LargeTests;
48 import org.apache.hadoop.hbase.MasterNotRunningException;
49 import org.apache.hadoop.hbase.MiniHBaseCluster;
50 import org.apache.hadoop.hbase.RegionTransition;
51 import org.apache.hadoop.hbase.Server;
52 import org.apache.hadoop.hbase.ServerName;
53 import org.apache.hadoop.hbase.TableName;
54 import org.apache.hadoop.hbase.UnknownRegionException;
55 import org.apache.hadoop.hbase.Waiter;
56 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
57 import org.apache.hadoop.hbase.catalog.MetaEditor;
58 import org.apache.hadoop.hbase.catalog.MetaReader;
59 import org.apache.hadoop.hbase.client.Delete;
60 import org.apache.hadoop.hbase.client.HBaseAdmin;
61 import org.apache.hadoop.hbase.client.HTable;
62 import org.apache.hadoop.hbase.client.Mutation;
63 import org.apache.hadoop.hbase.client.Put;
64 import org.apache.hadoop.hbase.client.Result;
65 import org.apache.hadoop.hbase.client.ResultScanner;
66 import org.apache.hadoop.hbase.client.Scan;
67 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
68 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
69 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
70 import org.apache.hadoop.hbase.exceptions.DeserializationException;
71 import org.apache.hadoop.hbase.executor.EventType;
72 import org.apache.hadoop.hbase.master.AssignmentManager;
73 import org.apache.hadoop.hbase.master.HMaster;
74 import org.apache.hadoop.hbase.master.RegionState;
75 import org.apache.hadoop.hbase.master.RegionStates;
76 import org.apache.hadoop.hbase.master.RegionState.State;
77 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
78 import org.apache.hadoop.hbase.util.Bytes;
79 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
80 import org.apache.hadoop.hbase.util.FSUtils;
81 import org.apache.hadoop.hbase.util.HBaseFsck;
82 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
83 import org.apache.hadoop.hbase.util.PairOfSameType;
84 import org.apache.hadoop.hbase.util.Threads;
85 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
86 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
87 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
88 import org.apache.zookeeper.KeeperException;
89 import org.apache.zookeeper.KeeperException.NodeExistsException;
90 import org.apache.zookeeper.data.Stat;
91 import org.junit.After;
92 import org.junit.AfterClass;
93 import org.junit.Assert;
94 import org.junit.Before;
95 import org.junit.BeforeClass;
96 import org.junit.Test;
97 import org.junit.experimental.categories.Category;
98
99 import com.google.protobuf.ServiceException;
100
101
102
103
104
105
106 @Category(LargeTests.class)
107 public class TestSplitTransactionOnCluster {
108 private static final Log LOG =
109 LogFactory.getLog(TestSplitTransactionOnCluster.class);
110 private HBaseAdmin admin = null;
111 private MiniHBaseCluster cluster = null;
112 private static final int NB_SERVERS = 3;
113 private static CountDownLatch latch = new CountDownLatch(1);
114 private static volatile boolean secondSplit = false;
115 private static volatile boolean callRollBack = false;
116 private static volatile boolean firstSplitCompleted = false;
117 private static boolean useZKForAssignment = true;
118
119 static final HBaseTestingUtility TESTING_UTIL =
120 new HBaseTestingUtility();
121
122 static void setupOnce() throws Exception {
123 TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
124 useZKForAssignment =
125 TESTING_UTIL.getConfiguration().getBoolean("hbase.assignment.usezk", false);
126 TESTING_UTIL.startMiniCluster(NB_SERVERS);
127 }
128
129 @BeforeClass public static void before() throws Exception {
130
131 TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
132 setupOnce();
133 }
134
135 @AfterClass public static void after() throws Exception {
136 TESTING_UTIL.shutdownMiniCluster();
137 }
138
139 @Before public void setup() throws IOException {
140 TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
141 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
142 this.cluster = TESTING_UTIL.getMiniHBaseCluster();
143 }
144
145 @After
146 public void tearDown() throws Exception {
147 this.admin.close();
148 }
149
150 private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
151 assertEquals(1, regions.size());
152 HRegionInfo hri = regions.get(0).getRegionInfo();
153 return waitOnRIT(hri);
154 }
155
156
157
158
159
160
161
162
163 private HRegionInfo waitOnRIT(final HRegionInfo hri) {
164
165
166 while (TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
167 getRegionStates().isRegionInTransition(hri)) {
168 LOG.info("Waiting on region in transition: " +
169 TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().
170 getRegionTransitionState(hri));
171 Threads.sleep(10);
172 }
173 return hri;
174 }
175
176 @SuppressWarnings("deprecation")
177 @Test(timeout = 60000)
178 public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
179 final TableName tableName =
180 TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
181
182 if (!useZKForAssignment) {
183
184 return;
185 }
186
187 try {
188
189 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
190 final List<HRegion> regions = cluster.getRegions(tableName);
191 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
192 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
193 final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
194 insertData(tableName.getName(), admin, t);
195 t.close();
196
197
198 this.admin.setBalancerRunning(false, true);
199
200 cluster.getMaster().setCatalogJanitorEnabled(false);
201
202
203 final HRegion region = findSplittableRegion(regions);
204 assertTrue("not able to find a splittable region", region != null);
205
206 new Thread() {
207 @Override
208 public void run() {
209 SplitTransaction st = null;
210 st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
211 try {
212 st.prepare();
213 st.execute(regionServer, regionServer);
214 } catch (IOException e) {
215
216 }
217 }
218 }.start();
219 for (int i = 0; !callRollBack && i < 100; i++) {
220 Thread.sleep(100);
221 }
222 assertTrue("Waited too long for rollback", callRollBack);
223 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
224 try {
225 secondSplit = true;
226
227 region.initialize();
228 st.prepare();
229 st.execute(regionServer, regionServer);
230 } catch (IOException e) {
231 LOG.debug("Rollback started :"+ e.getMessage());
232 st.rollback(regionServer, regionServer);
233 }
234 for (int i=0; !firstSplitCompleted && i<100; i++) {
235 Thread.sleep(100);
236 }
237 assertTrue("fist split did not complete", firstSplitCompleted);
238
239 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
240 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
241
242 for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
243 Thread.sleep(100);
244 }
245 assertFalse("region still in transition", rit.containsKey(
246 rit.containsKey(hri.getTable())));
247
248 List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
249
250 assertEquals("The parent region should be splitted", 2, onlineRegions.size());
251
252 List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
253 .getRegionStates().getRegionsOfTable(tableName);
254
255 assertEquals("No of regions in master", 2, regionsOfTable.size());
256 } finally {
257 admin.setBalancerRunning(true, false);
258 secondSplit = false;
259 firstSplitCompleted = false;
260 callRollBack = false;
261 cluster.getMaster().setCatalogJanitorEnabled(true);
262 TESTING_UTIL.deleteTable(tableName);
263 }
264 }
265
266 @Test(timeout = 60000)
267 public void testRITStateForRollback() throws Exception {
268 final TableName tableName =
269 TableName.valueOf("testRITStateForRollback");
270 try {
271
272 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
273 final List<HRegion> regions = cluster.getRegions(tableName);
274 final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
275 insertData(tableName.getName(), admin, t);
276 t.close();
277
278
279 this.admin.setBalancerRunning(false, true);
280
281 cluster.getMaster().setCatalogJanitorEnabled(false);
282
283
284 final HRegion region = findSplittableRegion(regions);
285 assertTrue("not able to find a splittable region", region != null);
286
287
288 region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
289 Coprocessor.PRIORITY_USER, region.getBaseConf());
290
291
292 this.admin.split(region.getRegionName(), new byte[] {42});
293
294
295 FailingSplitRegionObserver.latch.await();
296
297 LOG.info("Waiting for region to come out of RIT");
298 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
299 @Override
300 public boolean evaluate() throws Exception {
301 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
302 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
303 return !rit.containsKey(hri.getEncodedName());
304 }
305 });
306 } finally {
307 admin.setBalancerRunning(true, false);
308 cluster.getMaster().setCatalogJanitorEnabled(true);
309 TESTING_UTIL.deleteTable(tableName);
310 }
311 }
312
313 public static class FailingSplitRegionObserver extends BaseRegionObserver {
314 static volatile CountDownLatch latch = new CountDownLatch(1);
315 @Override
316 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
317 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
318 latch.countDown();
319 throw new IOException("Causing rollback of region split");
320 }
321 }
322
323
324
325
326
327
328
329
330
331
332
333
334 @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
335 throws IOException, InterruptedException, NodeExistsException, KeeperException,
336 DeserializationException, ServiceException {
337 final byte [] tableName =
338 Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
339
340
341 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
342 List<HRegion> regions = cluster.getRegions(tableName);
343 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
344
345 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
346
347
348 this.admin.setBalancerRunning(false, true);
349
350 cluster.getMaster().setCatalogJanitorEnabled(false);
351 try {
352
353 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
354
355 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
356 printOutRegions(server, "Initial regions: ");
357 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
358
359
360 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
361
362 split(hri, server, regionCount);
363
364 String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
365 hri.getEncodedName());
366 RegionTransition rt = null;
367 Stat stats = null;
368 List<HRegion> daughters = null;
369 if (useZKForAssignment) {
370 daughters = checkAndGetDaughters(tableName);
371
372
373 for (int i=0; i<100; i++) {
374 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
375 rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
376 hri.getEncodedName()));
377 if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
378 Thread.sleep(100);
379 }
380 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
381 assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
382
383 cluster.abortRegionServer(tableRegionIndex);
384 }
385 waitUntilRegionServerDead();
386 awaitDaughters(tableName, 2);
387 if (useZKForAssignment) {
388 regions = cluster.getRegions(tableName);
389 for (HRegion r: regions) {
390 assertTrue(daughters.contains(r));
391 }
392
393
394 for (int i=0; i<100; i++) {
395
396 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
397 if (stats == null) break;
398 Thread.sleep(100);
399 }
400 LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
401 assertTrue(stats == null);
402 }
403 } finally {
404
405 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
406 admin.setBalancerRunning(true, false);
407 cluster.getMaster().setCatalogJanitorEnabled(true);
408 cluster.startRegionServer();
409 t.close();
410 }
411 }
412
413 @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
414 throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
415 final byte [] tableName =
416 Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
417
418
419 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
420 List<HRegion> regions = cluster.getRegions(tableName);
421 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
422
423 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
424
425 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
426
427
428 this.admin.setBalancerRunning(false, true);
429
430 cluster.getMaster().setCatalogJanitorEnabled(false);
431 try {
432
433 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
434
435 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
436 printOutRegions(server, "Initial regions: ");
437 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
438
439
440 ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
441 if (useZKForAssignment) {
442 ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
443 hri, fakedServer);
444 } else {
445 regionStates.updateRegionState(hri, RegionState.State.CLOSING);
446 }
447
448
449 this.admin.split(hri.getRegionNameAsString());
450 this.admin.split(hri.getRegionNameAsString());
451 this.admin.split(hri.getRegionNameAsString());
452
453 for (int i = 0; i < 10; i++) {
454 Thread.sleep(100);
455 assertEquals(regionCount, ProtobufUtil.getOnlineRegions(server).size());
456 }
457 if (useZKForAssignment) {
458
459 ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
460 hri, fakedServer);
461 } else {
462 regionStates.regionOnline(hri, server.getServerName());
463 }
464
465 split(hri, server, regionCount);
466
467 checkAndGetDaughters(tableName);
468
469 } finally {
470 admin.setBalancerRunning(true, false);
471 cluster.getMaster().setCatalogJanitorEnabled(true);
472 t.close();
473 }
474 }
475
476
477
478
479
480
481
482 @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
483 throws IOException, InterruptedException, ServiceException {
484 final byte [] tableName =
485 Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
486
487
488 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
489 List<HRegion> regions = cluster.getRegions(tableName);
490 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
491
492 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
493
494
495 this.admin.setBalancerRunning(false, true);
496
497 cluster.getMaster().setCatalogJanitorEnabled(false);
498 try {
499
500 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
501
502 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
503 printOutRegions(server, "Initial regions: ");
504 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
505
506 split(hri, server, regionCount);
507
508 List<HRegion> daughters = checkAndGetDaughters(tableName);
509
510 regionCount = ProtobufUtil.getOnlineRegions(server).size();
511 HRegionInfo daughter = daughters.get(0).getRegionInfo();
512 LOG.info("Daughter we are going to split: " + daughter);
513
514
515 this.admin.compact(daughter.getRegionName());
516 daughters = cluster.getRegions(tableName);
517 HRegion daughterRegion = null;
518 for (HRegion r: daughters) {
519 if (r.getRegionInfo().equals(daughter)) {
520 daughterRegion = r;
521 LOG.info("Found matching HRI: " + daughterRegion);
522 break;
523 }
524 }
525 assertTrue(daughterRegion != null);
526 for (int i=0; i<100; i++) {
527 if (!daughterRegion.hasReferences()) break;
528 Threads.sleep(100);
529 }
530 assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
531 LOG.info("Daughter hri before split (has been compacted): " + daughter);
532 split(daughter, server, regionCount);
533
534 daughters = cluster.getRegions(tableName);
535 for (HRegion d: daughters) {
536 LOG.info("Regions before crash: " + d);
537 }
538
539 cluster.abortRegionServer(tableRegionIndex);
540 waitUntilRegionServerDead();
541 awaitDaughters(tableName, daughters.size());
542
543
544 regions = cluster.getRegions(tableName);
545 for (HRegion d: daughters) {
546 LOG.info("Regions after crash: " + d);
547 }
548 assertEquals(daughters.size(), regions.size());
549 for (HRegion r: regions) {
550 LOG.info("Regions post crash " + r);
551 assertTrue("Missing region post crash " + r, daughters.contains(r));
552 }
553 } finally {
554 admin.setBalancerRunning(true, false);
555 cluster.getMaster().setCatalogJanitorEnabled(true);
556 t.close();
557 }
558 }
559
560 @Test(timeout = 180000)
561 public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
562 Configuration conf = TESTING_UTIL.getConfiguration();
563 TableName userTableName =
564 TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
565 HTableDescriptor htd = new HTableDescriptor(userTableName);
566 HColumnDescriptor hcd = new HColumnDescriptor("col");
567 htd.addFamily(hcd);
568 admin.createTable(htd);
569 HTable table = new HTable(conf, userTableName);
570 try {
571 for (int i = 0; i <= 5; i++) {
572 String row = "row" + i;
573 Put p = new Put(row.getBytes());
574 String val = "Val" + i;
575 p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
576 table.put(p);
577 admin.flush(userTableName.getName());
578 Delete d = new Delete(row.getBytes());
579
580 table.delete(d);
581 admin.flush(userTableName.getName());
582 }
583 admin.majorCompact(userTableName.getName());
584 List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
585 .getMaster().getAssignmentManager().getRegionStates()
586 .getRegionsOfTable(userTableName);
587 HRegionInfo hRegionInfo = regionsOfTable.get(0);
588 Put p = new Put("row6".getBytes());
589 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
590 table.put(p);
591 p = new Put("row7".getBytes());
592 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
593 table.put(p);
594 p = new Put("row8".getBytes());
595 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
596 table.put(p);
597 admin.flush(userTableName.getName());
598 admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
599 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
600 .getAssignmentManager().getRegionStates()
601 .getRegionsOfTable(userTableName);
602
603 while (regionsOfTable.size() != 2) {
604 Thread.sleep(2000);
605 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
606 .getAssignmentManager().getRegionStates()
607 .getRegionsOfTable(userTableName);
608 }
609 Assert.assertEquals(2, regionsOfTable.size());
610 Scan s = new Scan();
611 ResultScanner scanner = table.getScanner(s);
612 int mainTableCount = 0;
613 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
614 mainTableCount++;
615 }
616 Assert.assertEquals(3, mainTableCount);
617 } finally {
618 table.close();
619 }
620 }
621
622
623
624
625 static class UselessTestAbortable implements Abortable {
626 boolean aborted = false;
627 @Override
628 public void abort(String why, Throwable e) {
629 LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
630 aborted = true;
631 }
632
633 @Override
634 public boolean isAborted() {
635 return this.aborted;
636 }
637 }
638
639
640
641
642
643
644
645
646
647
648
649 @Test(timeout = 400000)
650 public void testMasterRestartWhenSplittingIsPartial()
651 throws IOException, InterruptedException, NodeExistsException,
652 KeeperException, DeserializationException, ServiceException {
653 final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
654
655 if (!useZKForAssignment) {
656
657 return;
658 }
659
660
661 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
662 List<HRegion> regions = cluster.getRegions(tableName);
663 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
664
665 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
666
667
668 this.admin.setBalancerRunning(false, true);
669
670 cluster.getMaster().setCatalogJanitorEnabled(false);
671 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
672 "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
673 try {
674
675 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
676
677 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
678 printOutRegions(server, "Initial regions: ");
679
680
681 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
682
683
684 this.admin.split(hri.getRegionNameAsString());
685 checkAndGetDaughters(tableName);
686
687 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
688 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
689 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
690 + stats);
691 byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
692 RegionTransition rtd = RegionTransition.parseFrom(bytes);
693
694 assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
695 || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
696
697
698 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
699
700 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
701
702
703
704 hri.setOffline(true);
705 hri.setSplit(true);
706 ServerName regionServerOfRegion = master.getAssignmentManager()
707 .getRegionStates().getRegionServerOfRegion(hri);
708 assertTrue(regionServerOfRegion != null);
709
710
711 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
712 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
713 Stat stat = new Stat();
714 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
715
716 for (int i=0; data != null && i<60; i++) {
717 Thread.sleep(1000);
718 data = ZKUtil.getDataNoWatch(zkw, node, stat);
719 }
720 assertNull("Waited too long for ZK node to be removed: "+node, data);
721 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
722 assertTrue("Split parent should be in SPLIT state",
723 regionStates.isRegionInState(hri, State.SPLIT));
724 regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
725 assertTrue(regionServerOfRegion == null);
726 } finally {
727
728 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
729 admin.setBalancerRunning(true, false);
730 cluster.getMaster().setCatalogJanitorEnabled(true);
731 t.close();
732 zkw.close();
733 }
734 }
735
736
737
738
739
740
741
742
743
744 @Test (timeout = 300000)
745 public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
746 throws IOException, InterruptedException, NodeExistsException,
747 KeeperException, ServiceException {
748 final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
749
750
751 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
752 List<HRegion> regions = cluster.getRegions(tableName);
753 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
754
755 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
756
757
758 this.admin.setBalancerRunning(false, true);
759
760 cluster.getMaster().setCatalogJanitorEnabled(false);
761 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
762 "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
763 try {
764
765 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
766
767 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
768 printOutRegions(server, "Initial regions: ");
769
770 this.admin.split(hri.getRegionNameAsString());
771 checkAndGetDaughters(tableName);
772
773 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
774 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
775 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
776 + stats);
777 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
778 Stat stat = new Stat();
779 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
780
781 for (int i=0; data != null && i<60; i++) {
782 Thread.sleep(1000);
783 data = ZKUtil.getDataNoWatch(zkw, node, stat);
784 }
785 assertNull("Waited too long for ZK node to be removed: "+node, data);
786
787 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
788
789 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
790
791
792
793 hri.setOffline(true);
794 hri.setSplit(true);
795 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
796 assertTrue("Split parent should be in SPLIT state",
797 regionStates.isRegionInState(hri, State.SPLIT));
798 ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
799 assertTrue(regionServerOfRegion == null);
800 } finally {
801 this.admin.setBalancerRunning(true, false);
802 cluster.getMaster().setCatalogJanitorEnabled(true);
803 t.close();
804 zkw.close();
805 }
806 }
807
808
809
810
811
812
813
814
815
816
817
818
819 @Test(timeout = 60000)
820 public void testSplitBeforeSettingSplittingInZK() throws Exception,
821 InterruptedException, KeeperException {
822 testSplitBeforeSettingSplittingInZKInternals();
823 }
824
825 @Test(timeout = 60000)
826 public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
827 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
828 final TableName tableName =
829 TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
830
831 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
832 List<HRegion> regions = null;
833 try {
834 regions = cluster.getRegions(tableName);
835 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
836 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
837 insertData(tableName.getName(), admin, t);
838
839 admin.setBalancerRunning(false, true);
840
841 cluster.getMaster().setCatalogJanitorEnabled(false);
842 boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
843 tableName);
844 assertEquals("The specified table should present.", true, tableExists);
845 final HRegion region = findSplittableRegion(regions);
846 assertTrue("not able to find a splittable region", region != null);
847 SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
848 try {
849 st.prepare();
850 st.createDaughters(regionServer, regionServer);
851 } catch (IOException e) {
852
853 }
854 tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
855 tableName);
856 assertEquals("The specified table should present.", true, tableExists);
857 } finally {
858 if (regions != null) {
859 String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
860 .getEncodedName());
861 ZKUtil.deleteNodeFailSilent(zkw, node);
862 }
863 admin.setBalancerRunning(true, false);
864 cluster.getMaster().setCatalogJanitorEnabled(true);
865 t.close();
866 }
867 }
868
869 private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
870 InterruptedException {
871 Put p = new Put(Bytes.toBytes("row1"));
872 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
873 t.put(p);
874 p = new Put(Bytes.toBytes("row2"));
875 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
876 t.put(p);
877 p = new Put(Bytes.toBytes("row3"));
878 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
879 t.put(p);
880 p = new Put(Bytes.toBytes("row4"));
881 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
882 t.put(p);
883 admin.flush(tableName);
884 }
885
886
887
888
889
890 @Test(timeout = 60000)
891 public void testSplitRegionWithNoStoreFiles()
892 throws Exception {
893 final TableName tableName =
894 TableName.valueOf("testSplitRegionWithNoStoreFiles");
895
896 createTableAndWait(tableName.getName(), HConstants.CATALOG_FAMILY);
897 List<HRegion> regions = cluster.getRegions(tableName);
898 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
899 ensureTableRegionNotOnSameServerAsMeta(admin, hri);
900 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
901 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
902
903 this.admin.setBalancerRunning(false, true);
904
905 cluster.getMaster().setCatalogJanitorEnabled(false);
906 try {
907
908 printOutRegions(regionServer, "Initial regions: ");
909 Configuration conf = cluster.getConfiguration();
910 HBaseFsck.debugLsr(conf, new Path("/"));
911 Path rootDir = FSUtils.getRootDir(conf);
912 FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
913 Map<String, Path> storefiles =
914 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
915 assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
916
917
918 regions = cluster.getRegions(tableName);
919 final HRegion region = findSplittableRegion(regions);
920 assertTrue("not able to find a splittable region", region != null);
921
922
923 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
924 try {
925 st.prepare();
926 st.execute(regionServer, regionServer);
927 } catch (IOException e) {
928 fail("Split execution should have succeeded with no exceptions thrown");
929 }
930
931
932
933 List<HRegion> daughters = cluster.getRegions(tableName);
934 assertTrue(daughters.size() == 2);
935
936
937 HBaseFsck.debugLsr(conf, new Path("/"));
938 Map<String, Path> storefilesAfter =
939 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
940 assertEquals("Expected nothing but found " + storefilesAfter.toString(),
941 storefilesAfter.size(), 0);
942
943 hri = region.getRegionInfo();
944 AssignmentManager am = cluster.getMaster().getAssignmentManager();
945 RegionStates regionStates = am.getRegionStates();
946 long start = EnvironmentEdgeManager.currentTimeMillis();
947 while (!regionStates.isRegionInState(hri, State.SPLIT)) {
948 assertFalse("Timed out in waiting split parent to be in state SPLIT",
949 EnvironmentEdgeManager.currentTimeMillis() - start > 60000);
950 Thread.sleep(500);
951 }
952
953
954 am.assign(hri, true, true);
955 assertFalse("Split region can't be assigned",
956 regionStates.isRegionInTransition(hri));
957 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
958
959
960 am.unassign(hri, true, null);
961 assertFalse("Split region can't be unassigned",
962 regionStates.isRegionInTransition(hri));
963 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
964 } finally {
965 admin.setBalancerRunning(true, false);
966 cluster.getMaster().setCatalogJanitorEnabled(true);
967 }
968 }
969
970 @Test(timeout = 180000)
971 public void testSplitHooksBeforeAndAfterPONR() throws Exception {
972 TableName firstTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1");
973 TableName secondTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2");
974 HColumnDescriptor hcd = new HColumnDescriptor("cf");
975
976 HTableDescriptor desc = new HTableDescriptor(firstTable);
977 desc.addCoprocessor(MockedRegionObserver.class.getName());
978 desc.addFamily(hcd);
979 admin.createTable(desc);
980 TESTING_UTIL.waitUntilAllRegionsAssigned(firstTable);
981
982 desc = new HTableDescriptor(secondTable);
983 desc.addFamily(hcd);
984 admin.createTable(desc);
985 TESTING_UTIL.waitUntilAllRegionsAssigned(secondTable);
986
987 List<HRegion> firstTableRegions = cluster.getRegions(firstTable);
988 List<HRegion> secondTableRegions = cluster.getRegions(secondTable);
989
990
991 if (firstTableRegions.size() == 0 || secondTableRegions.size() == 0) {
992 fail("Each table should have at least one region.");
993 }
994 ServerName serverName =
995 cluster.getServerHoldingRegion(firstTableRegions.get(0).getRegionName());
996 admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
997 Bytes.toBytes(serverName.getServerName()));
998 HTable table1 = null;
999 HTable table2 = null;
1000 try {
1001 table1 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1002 table2 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1003 insertData(firstTable.getName(), admin, table1);
1004 insertData(secondTable.getName(), admin, table2);
1005 admin.split(firstTable.getName(), "row2".getBytes());
1006 firstTableRegions = cluster.getRegions(firstTable.getName());
1007 while (firstTableRegions.size() != 2) {
1008 Thread.sleep(1000);
1009 firstTableRegions = cluster.getRegions(firstTable.getName());
1010 }
1011 assertEquals("Number of regions after split should be 2.", 2, firstTableRegions.size());
1012 secondTableRegions = cluster.getRegions(secondTable.getName());
1013 assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
1014 } finally {
1015 if (table1 != null) {
1016 table1.close();
1017 }
1018 if (table2 != null) {
1019 table2.close();
1020 }
1021 TESTING_UTIL.deleteTable(firstTable);
1022 TESTING_UTIL.deleteTable(secondTable);
1023 }
1024 }
1025
1026 private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
1027 final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
1028 try {
1029
1030 createTableAndWait(tableName, Bytes.toBytes("cf"));
1031
1032 List<HRegion> regions = awaitTableRegions(tableName);
1033 assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
1034
1035 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
1036 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1037 final HRegion region = findSplittableRegion(regions);
1038 assertTrue("not able to find a splittable region", region != null);
1039 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
1040 @Override
1041 public PairOfSameType<HRegion> stepsBeforePONR(final Server server,
1042 final RegionServerServices services, boolean testing) throws IOException {
1043 throw new SplittingNodeCreationFailedException ();
1044 }
1045 };
1046 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1047 region.getRegionInfo().getEncodedName());
1048 regionServer.getZooKeeper().sync(node);
1049 for (int i = 0; i < 100; i++) {
1050
1051
1052
1053 if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
1054 Thread.sleep(100);
1055 }
1056 }
1057 try {
1058 st.prepare();
1059 st.execute(regionServer, regionServer);
1060 } catch (IOException e) {
1061
1062
1063
1064 assertTrue("Should be instance of CreateSplittingNodeFailedException",
1065 e instanceof SplittingNodeCreationFailedException );
1066 node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1067 region.getRegionInfo().getEncodedName());
1068 {
1069 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1070 }
1071 assertTrue(st.rollback(regionServer, regionServer));
1072 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1073 }
1074 } finally {
1075 TESTING_UTIL.deleteTable(tableName);
1076 }
1077 }
1078
1079 public static class MockedSplitTransaction extends SplitTransaction {
1080
1081 private HRegion currentRegion;
1082 public MockedSplitTransaction(HRegion r, byte[] splitrow) {
1083 super(r, splitrow);
1084 this.currentRegion = r;
1085 }
1086
1087 @Override
1088 void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b)
1089 throws IOException {
1090 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1091 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1092 try {
1093 if (!secondSplit){
1094 callRollBack = true;
1095 latch.await();
1096 }
1097 } catch (InterruptedException e) {
1098 }
1099
1100 }
1101 super.transitionZKNode(server, services, a, b);
1102 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1103 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1104 firstSplitCompleted = true;
1105 }
1106 }
1107 @Override
1108 public boolean rollback(Server server, RegionServerServices services) throws IOException {
1109 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1110 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1111 if(secondSplit){
1112 super.rollback(server, services);
1113 latch.countDown();
1114 return true;
1115 }
1116 }
1117 return super.rollback(server, services);
1118 }
1119
1120 }
1121
1122 private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
1123 for (int i = 0; i < 5; ++i) {
1124 for (HRegion r: regions) {
1125 if (r.isSplittable()) {
1126 return(r);
1127 }
1128 }
1129 Thread.sleep(100);
1130 }
1131 return(null);
1132 }
1133
1134 private List<HRegion> checkAndGetDaughters(byte[] tableName)
1135 throws InterruptedException {
1136 List<HRegion> daughters = null;
1137
1138 for (int i=0; i<100; i++) {
1139 daughters = cluster.getRegions(tableName);
1140 if (daughters.size() >= 2) break;
1141 Thread.sleep(100);
1142 }
1143 assertTrue(daughters.size() >= 2);
1144 return daughters;
1145 }
1146
1147 private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
1148 throws IOException, InterruptedException {
1149 cluster.abortMaster(0);
1150 cluster.waitOnMaster(0);
1151 cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
1152 MockMasterWithoutCatalogJanitor.class, HMaster.class);
1153 MockMasterWithoutCatalogJanitor master = null;
1154 master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
1155 cluster.waitForActiveAndReadyMaster();
1156 return master;
1157 }
1158
1159 private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
1160 throws IOException, InterruptedException {
1161 this.admin.split(hri.getRegionNameAsString());
1162 try {
1163 for (int i = 0; ProtobufUtil.getOnlineRegions(server).size() <= regionCount && i < 300; i++) {
1164 LOG.debug("Waiting on region to split");
1165 Thread.sleep(100);
1166 }
1167
1168 assertFalse("Waited too long for split",
1169 ProtobufUtil.getOnlineRegions(server).size() <= regionCount);
1170 } catch (RegionServerStoppedException e) {
1171 if (useZKForAssignment) {
1172
1173 LOG.error(e);
1174 throw e;
1175 }
1176 }
1177 }
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190 private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
1191 final HRegionInfo hri)
1192 throws HBaseIOException, MasterNotRunningException,
1193 ZooKeeperConnectionException, InterruptedException {
1194
1195
1196
1197 int metaServerIndex = cluster.getServerWithMeta();
1198 assertTrue(metaServerIndex != -1);
1199 HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1200 int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1201 assertTrue(tableRegionIndex != -1);
1202 HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1203 if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1204 HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1205 assertNotNull(hrs);
1206 assertNotNull(hri);
1207 LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1208 metaRegionServer.getServerName() + " to " +
1209 hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1210 admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1211 }
1212
1213 for (int i = 0; i < 100; i++) {
1214 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1215 if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1216 LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1217 tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1218 Thread.sleep(100);
1219 }
1220 assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1221 && tableRegionIndex != metaServerIndex);
1222
1223 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1224 assertTrue(tableRegionIndex != -1);
1225 assertNotSame(metaServerIndex, tableRegionIndex);
1226 return tableRegionIndex;
1227 }
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238 private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1239 final HRegionServer notThisOne) {
1240 for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1241 HRegionServer hrs = rst.getRegionServer();
1242 if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1243 if (hrs.isStopping() || hrs.isStopped()) continue;
1244 return hrs;
1245 }
1246 return null;
1247 }
1248
1249 private void printOutRegions(final HRegionServer hrs, final String prefix)
1250 throws IOException {
1251 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
1252 for (HRegionInfo region: regions) {
1253 LOG.info(prefix + region.getRegionNameAsString());
1254 }
1255 }
1256
1257 private void waitUntilRegionServerDead() throws InterruptedException {
1258
1259 for (int i=0; cluster.getMaster().getClusterStatus().
1260 getServers().size() == NB_SERVERS && i<100; i++) {
1261 LOG.info("Waiting on server to go down");
1262 Thread.sleep(100);
1263 }
1264 assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1265 getServers().size() == NB_SERVERS);
1266 }
1267
1268 private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
1269
1270 for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1271 LOG.info("Waiting for repair to happen");
1272 Thread.sleep(1000);
1273 }
1274 if (cluster.getRegions(tableName).size() < numDaughters) {
1275 fail("Waiting too long for daughter regions");
1276 }
1277 }
1278
1279 private List<HRegion> awaitTableRegions(final byte[] tableName) throws InterruptedException {
1280 List<HRegion> regions = null;
1281 for (int i = 0; i < 100; i++) {
1282 regions = cluster.getRegions(tableName);
1283 if (regions.size() > 0) break;
1284 Thread.sleep(100);
1285 }
1286 return regions;
1287 }
1288
1289 private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException,
1290 InterruptedException {
1291 HTable t = TESTING_UTIL.createTable(tableName, cf);
1292 awaitTableRegions(tableName);
1293 assertTrue("Table not online: " + Bytes.toString(tableName),
1294 cluster.getRegions(tableName).size() != 0);
1295 return t;
1296 }
1297
1298 public static class MockMasterWithoutCatalogJanitor extends HMaster {
1299
1300 public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException,
1301 InterruptedException {
1302 super(conf);
1303 }
1304 }
1305
1306 private static class SplittingNodeCreationFailedException extends IOException {
1307 private static final long serialVersionUID = 1652404976265623004L;
1308
1309 public SplittingNodeCreationFailedException () {
1310 super();
1311 }
1312 }
1313
1314 public static class MockedRegionObserver extends BaseRegionObserver {
1315 private SplitTransaction st = null;
1316 private PairOfSameType<HRegion> daughterRegions = null;
1317
1318 @Override
1319 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
1320 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
1321 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1322 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1323 List<HRegion> onlineRegions =
1324 rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
1325 HRegion region = onlineRegions.get(0);
1326 for (HRegion r : onlineRegions) {
1327 if (r.getRegionInfo().containsRow(splitKey)) {
1328 region = r;
1329 break;
1330 }
1331 }
1332 st = new SplitTransaction(region, splitKey);
1333 if (!st.prepare()) {
1334 LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
1335 + " failed. So returning null. ");
1336 ctx.bypass();
1337 return;
1338 }
1339 region.forceSplit(splitKey);
1340 daughterRegions = st.stepsBeforePONR(rs, rs, false);
1341 HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
1342 copyOfParent.setOffline(true);
1343 copyOfParent.setSplit(true);
1344
1345 Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
1346 MetaEditor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
1347 daughterRegions.getSecond().getRegionInfo());
1348 metaEntries.add(putParent);
1349
1350 Put putA = MetaEditor.makePutFromRegionInfo(daughterRegions.getFirst().getRegionInfo());
1351 Put putB = MetaEditor.makePutFromRegionInfo(daughterRegions.getSecond().getRegionInfo());
1352 st.addLocation(putA, rs.getServerName(), 1);
1353 st.addLocation(putB, rs.getServerName(), 1);
1354 metaEntries.add(putA);
1355 metaEntries.add(putB);
1356 }
1357
1358 @Override
1359 public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
1360 throws IOException {
1361 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1362 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1363 st.stepsAfterPONR(rs, rs, daughterRegions);
1364 }
1365
1366 }
1367 }
1368