1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNotSame;
25 import static org.junit.Assert.assertNull;
26 import static org.junit.Assert.assertTrue;
27 import static org.junit.Assert.fail;
28
29 import java.io.IOException;
30 import java.util.Collection;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.concurrent.CountDownLatch;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.Abortable;
41 import org.apache.hadoop.hbase.Coprocessor;
42 import org.apache.hadoop.hbase.HBaseIOException;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.HRegionInfo;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.testclassification.LargeTests;
49 import org.apache.hadoop.hbase.MasterNotRunningException;
50 import org.apache.hadoop.hbase.MiniHBaseCluster;
51 import org.apache.hadoop.hbase.RegionTransition;
52 import org.apache.hadoop.hbase.Server;
53 import org.apache.hadoop.hbase.ServerName;
54 import org.apache.hadoop.hbase.TableName;
55 import org.apache.hadoop.hbase.UnknownRegionException;
56 import org.apache.hadoop.hbase.Waiter;
57 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
58 import org.apache.hadoop.hbase.catalog.MetaEditor;
59 import org.apache.hadoop.hbase.catalog.MetaReader;
60 import org.apache.hadoop.hbase.client.Delete;
61 import org.apache.hadoop.hbase.client.HBaseAdmin;
62 import org.apache.hadoop.hbase.client.HTable;
63 import org.apache.hadoop.hbase.client.Mutation;
64 import org.apache.hadoop.hbase.client.Put;
65 import org.apache.hadoop.hbase.client.Result;
66 import org.apache.hadoop.hbase.client.ResultScanner;
67 import org.apache.hadoop.hbase.client.Scan;
68 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
69 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
70 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
71 import org.apache.hadoop.hbase.exceptions.DeserializationException;
72 import org.apache.hadoop.hbase.executor.EventType;
73 import org.apache.hadoop.hbase.master.AssignmentManager;
74 import org.apache.hadoop.hbase.master.HMaster;
75 import org.apache.hadoop.hbase.master.RegionState;
76 import org.apache.hadoop.hbase.master.RegionStates;
77 import org.apache.hadoop.hbase.master.RegionState.State;
78 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
79 import org.apache.hadoop.hbase.util.Bytes;
80 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
81 import org.apache.hadoop.hbase.util.FSUtils;
82 import org.apache.hadoop.hbase.util.HBaseFsck;
83 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
84 import org.apache.hadoop.hbase.util.PairOfSameType;
85 import org.apache.hadoop.hbase.util.Threads;
86 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
87 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
88 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
89 import org.apache.zookeeper.KeeperException;
90 import org.apache.zookeeper.KeeperException.NodeExistsException;
91 import org.apache.zookeeper.data.Stat;
92 import org.junit.After;
93 import org.junit.AfterClass;
94 import org.junit.Assert;
95 import org.junit.Before;
96 import org.junit.BeforeClass;
97 import org.junit.Test;
98 import org.junit.experimental.categories.Category;
99
100 import com.google.protobuf.ServiceException;
101
102
103
104
105
106
107 @Category(LargeTests.class)
108 public class TestSplitTransactionOnCluster {
109 private static final Log LOG =
110 LogFactory.getLog(TestSplitTransactionOnCluster.class);
111 private HBaseAdmin admin = null;
112 private MiniHBaseCluster cluster = null;
113 private static final int NB_SERVERS = 3;
114 private static CountDownLatch latch = new CountDownLatch(1);
115 private static volatile boolean secondSplit = false;
116 private static volatile boolean callRollBack = false;
117 private static volatile boolean firstSplitCompleted = false;
118 private static boolean useZKForAssignment = true;
119
120 static final HBaseTestingUtility TESTING_UTIL =
121 new HBaseTestingUtility();
122
123 static void setupOnce() throws Exception {
124 TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
125 useZKForAssignment =
126 TESTING_UTIL.getConfiguration().getBoolean("hbase.assignment.usezk", false);
127 TESTING_UTIL.startMiniCluster(NB_SERVERS);
128 }
129
130 @BeforeClass public static void before() throws Exception {
131
132 TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
133 setupOnce();
134 }
135
136 @AfterClass public static void after() throws Exception {
137 TESTING_UTIL.shutdownMiniCluster();
138 }
139
140 @Before public void setup() throws IOException {
141 TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
142 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
143 this.cluster = TESTING_UTIL.getMiniHBaseCluster();
144 }
145
146 @After
147 public void tearDown() throws Exception {
148 this.admin.close();
149 }
150
151 private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
152 assertEquals(1, regions.size());
153 HRegionInfo hri = regions.get(0).getRegionInfo();
154 return waitOnRIT(hri);
155 }
156
157
158
159
160
161
162
163
164 private HRegionInfo waitOnRIT(final HRegionInfo hri) {
165
166
167 while (TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
168 getRegionStates().isRegionInTransition(hri)) {
169 LOG.info("Waiting on region in transition: " +
170 TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().
171 getRegionTransitionState(hri));
172 Threads.sleep(10);
173 }
174 return hri;
175 }
176
177 @SuppressWarnings("deprecation")
178 @Test(timeout = 60000)
179 public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
180 final TableName tableName =
181 TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
182
183 if (!useZKForAssignment) {
184
185 return;
186 }
187
188 try {
189
190 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
191 final List<HRegion> regions = cluster.getRegions(tableName);
192 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
193 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
194 final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
195 insertData(tableName.getName(), admin, t);
196 t.close();
197
198
199 this.admin.setBalancerRunning(false, true);
200
201 cluster.getMaster().setCatalogJanitorEnabled(false);
202
203
204 final HRegion region = findSplittableRegion(regions);
205 assertTrue("not able to find a splittable region", region != null);
206
207 new Thread() {
208 @Override
209 public void run() {
210 SplitTransaction st = null;
211 st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
212 try {
213 st.prepare();
214 st.execute(regionServer, regionServer);
215 } catch (IOException e) {
216
217 }
218 }
219 }.start();
220 for (int i = 0; !callRollBack && i < 100; i++) {
221 Thread.sleep(100);
222 }
223 assertTrue("Waited too long for rollback", callRollBack);
224 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
225 try {
226 secondSplit = true;
227
228 region.initialize();
229 st.prepare();
230 st.execute(regionServer, regionServer);
231 } catch (IOException e) {
232 LOG.debug("Rollback started :"+ e.getMessage());
233 st.rollback(regionServer, regionServer);
234 }
235 for (int i=0; !firstSplitCompleted && i<100; i++) {
236 Thread.sleep(100);
237 }
238 assertTrue("fist split did not complete", firstSplitCompleted);
239
240 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
241 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
242
243 for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
244 Thread.sleep(100);
245 }
246 assertFalse("region still in transition", rit.containsKey(
247 rit.containsKey(hri.getTable())));
248
249 List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
250
251 assertEquals("The parent region should be splitted", 2, onlineRegions.size());
252
253 List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
254 .getRegionStates().getRegionsOfTable(tableName);
255
256 assertEquals("No of regions in master", 2, regionsOfTable.size());
257 } finally {
258 admin.setBalancerRunning(true, false);
259 secondSplit = false;
260 firstSplitCompleted = false;
261 callRollBack = false;
262 cluster.getMaster().setCatalogJanitorEnabled(true);
263 TESTING_UTIL.deleteTable(tableName);
264 }
265 }
266
267 @Test(timeout = 60000)
268 public void testRITStateForRollback() throws Exception {
269 final TableName tableName =
270 TableName.valueOf("testRITStateForRollback");
271 try {
272
273 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
274 final List<HRegion> regions = cluster.getRegions(tableName);
275 final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
276 insertData(tableName.getName(), admin, t);
277 t.close();
278
279
280 this.admin.setBalancerRunning(false, true);
281
282 cluster.getMaster().setCatalogJanitorEnabled(false);
283
284
285 final HRegion region = findSplittableRegion(regions);
286 assertTrue("not able to find a splittable region", region != null);
287
288
289 region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
290 Coprocessor.PRIORITY_USER, region.getBaseConf());
291
292
293 this.admin.split(region.getRegionName(), new byte[] {42});
294
295
296 FailingSplitRegionObserver.latch.await();
297
298 LOG.info("Waiting for region to come out of RIT");
299 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
300 @Override
301 public boolean evaluate() throws Exception {
302 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
303 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
304 return !rit.containsKey(hri.getEncodedName());
305 }
306 });
307 } finally {
308 admin.setBalancerRunning(true, false);
309 cluster.getMaster().setCatalogJanitorEnabled(true);
310 TESTING_UTIL.deleteTable(tableName);
311 }
312 }
313
314 public static class FailingSplitRegionObserver extends BaseRegionObserver {
315 static volatile CountDownLatch latch = new CountDownLatch(1);
316 @Override
317 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
318 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
319 latch.countDown();
320 throw new IOException("Causing rollback of region split");
321 }
322 }
323
324
325
326
327
328
329
330
331
332
333
334
335 @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
336 throws IOException, InterruptedException, NodeExistsException, KeeperException,
337 DeserializationException, ServiceException {
338 final byte [] tableName =
339 Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
340
341
342 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
343 List<HRegion> regions = cluster.getRegions(tableName);
344 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
345
346 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
347
348
349 this.admin.setBalancerRunning(false, true);
350
351 cluster.getMaster().setCatalogJanitorEnabled(false);
352 try {
353
354 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
355
356 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
357 printOutRegions(server, "Initial regions: ");
358 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
359
360
361 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
362
363 split(hri, server, regionCount);
364
365 String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
366 hri.getEncodedName());
367 RegionTransition rt = null;
368 Stat stats = null;
369 List<HRegion> daughters = null;
370 if (useZKForAssignment) {
371 daughters = checkAndGetDaughters(tableName);
372
373
374 for (int i=0; i<100; i++) {
375 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
376 rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
377 hri.getEncodedName()));
378 if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
379 Thread.sleep(100);
380 }
381 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
382 assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
383
384 cluster.abortRegionServer(tableRegionIndex);
385 }
386 waitUntilRegionServerDead();
387 awaitDaughters(tableName, 2);
388 if (useZKForAssignment) {
389 regions = cluster.getRegions(tableName);
390 for (HRegion r: regions) {
391 assertTrue(daughters.contains(r));
392 }
393
394
395 for (int i=0; i<100; i++) {
396
397 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
398 if (stats == null) break;
399 Thread.sleep(100);
400 }
401 LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
402 assertTrue(stats == null);
403 }
404 } finally {
405
406 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
407 admin.setBalancerRunning(true, false);
408 cluster.getMaster().setCatalogJanitorEnabled(true);
409 cluster.startRegionServer();
410 t.close();
411 }
412 }
413
414 @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
415 throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
416 final byte [] tableName =
417 Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
418
419
420 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
421 List<HRegion> regions = cluster.getRegions(tableName);
422 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
423
424 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
425
426 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
427
428
429 this.admin.setBalancerRunning(false, true);
430
431 cluster.getMaster().setCatalogJanitorEnabled(false);
432 try {
433
434 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
435
436 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
437 printOutRegions(server, "Initial regions: ");
438 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
439
440
441 ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
442 if (useZKForAssignment) {
443 ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
444 hri, fakedServer);
445 } else {
446 regionStates.updateRegionState(hri, RegionState.State.CLOSING);
447 }
448
449
450 this.admin.split(hri.getRegionNameAsString());
451 this.admin.split(hri.getRegionNameAsString());
452 this.admin.split(hri.getRegionNameAsString());
453
454 for (int i = 0; i < 10; i++) {
455 Thread.sleep(100);
456 assertEquals(regionCount, ProtobufUtil.getOnlineRegions(server).size());
457 }
458 if (useZKForAssignment) {
459
460 ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
461 hri, fakedServer);
462 } else {
463 regionStates.regionOnline(hri, server.getServerName());
464 }
465
466 split(hri, server, regionCount);
467
468 checkAndGetDaughters(tableName);
469
470 } finally {
471 admin.setBalancerRunning(true, false);
472 cluster.getMaster().setCatalogJanitorEnabled(true);
473 t.close();
474 }
475 }
476
477
478
479
480
481
482
483 @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
484 throws IOException, InterruptedException, ServiceException {
485 final byte [] tableName =
486 Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
487
488
489 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
490 List<HRegion> regions = cluster.getRegions(tableName);
491 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
492
493 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
494
495
496 this.admin.setBalancerRunning(false, true);
497
498 cluster.getMaster().setCatalogJanitorEnabled(false);
499 try {
500
501 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
502
503 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
504 printOutRegions(server, "Initial regions: ");
505 int regionCount = ProtobufUtil.getOnlineRegions(server).size();
506
507 split(hri, server, regionCount);
508
509 List<HRegion> daughters = checkAndGetDaughters(tableName);
510
511 regionCount = ProtobufUtil.getOnlineRegions(server).size();
512 HRegionInfo daughter = daughters.get(0).getRegionInfo();
513 LOG.info("Daughter we are going to split: " + daughter);
514
515
516 this.admin.compact(daughter.getRegionName());
517 daughters = cluster.getRegions(tableName);
518 HRegion daughterRegion = null;
519 for (HRegion r: daughters) {
520 if (r.getRegionInfo().equals(daughter)) {
521 daughterRegion = r;
522 LOG.info("Found matching HRI: " + daughterRegion);
523 break;
524 }
525 }
526 assertTrue(daughterRegion != null);
527 for (int i=0; i<100; i++) {
528 if (!daughterRegion.hasReferences()) break;
529 Threads.sleep(100);
530 }
531 assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
532 LOG.info("Daughter hri before split (has been compacted): " + daughter);
533 split(daughter, server, regionCount);
534
535 daughters = cluster.getRegions(tableName);
536 for (HRegion d: daughters) {
537 LOG.info("Regions before crash: " + d);
538 }
539
540 cluster.abortRegionServer(tableRegionIndex);
541 waitUntilRegionServerDead();
542 awaitDaughters(tableName, daughters.size());
543
544
545 regions = cluster.getRegions(tableName);
546 for (HRegion d: daughters) {
547 LOG.info("Regions after crash: " + d);
548 }
549 assertEquals(daughters.size(), regions.size());
550 for (HRegion r: regions) {
551 LOG.info("Regions post crash " + r);
552 assertTrue("Missing region post crash " + r, daughters.contains(r));
553 }
554 } finally {
555 admin.setBalancerRunning(true, false);
556 cluster.getMaster().setCatalogJanitorEnabled(true);
557 t.close();
558 }
559 }
560
561 @Test(timeout = 180000)
562 public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
563 Configuration conf = TESTING_UTIL.getConfiguration();
564 TableName userTableName =
565 TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
566 HTableDescriptor htd = new HTableDescriptor(userTableName);
567 HColumnDescriptor hcd = new HColumnDescriptor("col");
568 htd.addFamily(hcd);
569 admin.createTable(htd);
570 HTable table = new HTable(conf, userTableName);
571 try {
572 for (int i = 0; i <= 5; i++) {
573 String row = "row" + i;
574 Put p = new Put(row.getBytes());
575 String val = "Val" + i;
576 p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
577 table.put(p);
578 admin.flush(userTableName.getName());
579 Delete d = new Delete(row.getBytes());
580
581 table.delete(d);
582 admin.flush(userTableName.getName());
583 }
584 admin.majorCompact(userTableName.getName());
585 List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
586 .getMaster().getAssignmentManager().getRegionStates()
587 .getRegionsOfTable(userTableName);
588 HRegionInfo hRegionInfo = regionsOfTable.get(0);
589 Put p = new Put("row6".getBytes());
590 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
591 table.put(p);
592 p = new Put("row7".getBytes());
593 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
594 table.put(p);
595 p = new Put("row8".getBytes());
596 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
597 table.put(p);
598 admin.flush(userTableName.getName());
599 admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
600 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
601 .getAssignmentManager().getRegionStates()
602 .getRegionsOfTable(userTableName);
603
604 while (regionsOfTable.size() != 2) {
605 Thread.sleep(2000);
606 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
607 .getAssignmentManager().getRegionStates()
608 .getRegionsOfTable(userTableName);
609 }
610 Assert.assertEquals(2, regionsOfTable.size());
611 Scan s = new Scan();
612 ResultScanner scanner = table.getScanner(s);
613 int mainTableCount = 0;
614 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
615 mainTableCount++;
616 }
617 Assert.assertEquals(3, mainTableCount);
618 } finally {
619 table.close();
620 }
621 }
622
623
624
625
626 static class UselessTestAbortable implements Abortable {
627 boolean aborted = false;
628 @Override
629 public void abort(String why, Throwable e) {
630 LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
631 aborted = true;
632 }
633
634 @Override
635 public boolean isAborted() {
636 return this.aborted;
637 }
638 }
639
640
641
642
643
644
645
646
647
648
649
650 @Test(timeout = 400000)
651 public void testMasterRestartWhenSplittingIsPartial()
652 throws IOException, InterruptedException, NodeExistsException,
653 KeeperException, DeserializationException, ServiceException {
654 final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
655
656 if (!useZKForAssignment) {
657
658 return;
659 }
660
661
662 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
663 List<HRegion> regions = cluster.getRegions(tableName);
664 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
665
666 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
667
668
669 this.admin.setBalancerRunning(false, true);
670
671 cluster.getMaster().setCatalogJanitorEnabled(false);
672 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
673 "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
674 try {
675
676 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
677
678 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
679 printOutRegions(server, "Initial regions: ");
680
681
682 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
683
684
685 this.admin.split(hri.getRegionNameAsString());
686 checkAndGetDaughters(tableName);
687
688 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
689 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
690 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
691 + stats);
692 byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
693 RegionTransition rtd = RegionTransition.parseFrom(bytes);
694
695 assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
696 || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
697
698
699 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
700
701 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
702
703
704
705 hri.setOffline(true);
706 hri.setSplit(true);
707 ServerName regionServerOfRegion = master.getAssignmentManager()
708 .getRegionStates().getRegionServerOfRegion(hri);
709 assertTrue(regionServerOfRegion != null);
710
711
712 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
713 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
714 Stat stat = new Stat();
715 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
716
717 for (int i=0; data != null && i<60; i++) {
718 Thread.sleep(1000);
719 data = ZKUtil.getDataNoWatch(zkw, node, stat);
720 }
721 assertNull("Waited too long for ZK node to be removed: "+node, data);
722 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
723 assertTrue("Split parent should be in SPLIT state",
724 regionStates.isRegionInState(hri, State.SPLIT));
725 regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
726 assertTrue(regionServerOfRegion == null);
727 } finally {
728
729 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
730 admin.setBalancerRunning(true, false);
731 cluster.getMaster().setCatalogJanitorEnabled(true);
732 t.close();
733 zkw.close();
734 }
735 }
736
737
738
739
740
741
742
743
744
745 @Test (timeout = 300000)
746 public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
747 throws IOException, InterruptedException, NodeExistsException,
748 KeeperException, ServiceException {
749 final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
750
751
752 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
753 List<HRegion> regions = cluster.getRegions(tableName);
754 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
755
756 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
757
758
759 this.admin.setBalancerRunning(false, true);
760
761 cluster.getMaster().setCatalogJanitorEnabled(false);
762 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
763 "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
764 try {
765
766 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
767
768 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
769 printOutRegions(server, "Initial regions: ");
770
771 this.admin.split(hri.getRegionNameAsString());
772 checkAndGetDaughters(tableName);
773
774 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
775 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
776 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
777 + stats);
778 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
779 Stat stat = new Stat();
780 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
781
782 for (int i=0; data != null && i<60; i++) {
783 Thread.sleep(1000);
784 data = ZKUtil.getDataNoWatch(zkw, node, stat);
785 }
786 assertNull("Waited too long for ZK node to be removed: "+node, data);
787
788 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
789
790 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
791
792
793
794 hri.setOffline(true);
795 hri.setSplit(true);
796 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
797 assertTrue("Split parent should be in SPLIT state",
798 regionStates.isRegionInState(hri, State.SPLIT));
799 ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
800 assertTrue(regionServerOfRegion == null);
801 } finally {
802 this.admin.setBalancerRunning(true, false);
803 cluster.getMaster().setCatalogJanitorEnabled(true);
804 t.close();
805 zkw.close();
806 }
807 }
808
809
810
811
812
813
814
815
816
817
818
819
820 @Test(timeout = 60000)
821 public void testSplitBeforeSettingSplittingInZK() throws Exception,
822 InterruptedException, KeeperException {
823 testSplitBeforeSettingSplittingInZKInternals();
824 }
825
826 @Test(timeout = 60000)
827 public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
828 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
829 final TableName tableName =
830 TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
831
832 HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
833 List<HRegion> regions = null;
834 try {
835 regions = cluster.getRegions(tableName);
836 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
837 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
838 insertData(tableName.getName(), admin, t);
839
840 admin.setBalancerRunning(false, true);
841
842 cluster.getMaster().setCatalogJanitorEnabled(false);
843 boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
844 tableName);
845 assertEquals("The specified table should present.", true, tableExists);
846 final HRegion region = findSplittableRegion(regions);
847 assertTrue("not able to find a splittable region", region != null);
848 SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
849 try {
850 st.prepare();
851 st.createDaughters(regionServer, regionServer);
852 } catch (IOException e) {
853
854 }
855 tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
856 tableName);
857 assertEquals("The specified table should present.", true, tableExists);
858 } finally {
859 if (regions != null) {
860 String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
861 .getEncodedName());
862 ZKUtil.deleteNodeFailSilent(zkw, node);
863 }
864 admin.setBalancerRunning(true, false);
865 cluster.getMaster().setCatalogJanitorEnabled(true);
866 t.close();
867 }
868 }
869
870 private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
871 InterruptedException {
872 Put p = new Put(Bytes.toBytes("row1"));
873 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
874 t.put(p);
875 p = new Put(Bytes.toBytes("row2"));
876 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
877 t.put(p);
878 p = new Put(Bytes.toBytes("row3"));
879 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
880 t.put(p);
881 p = new Put(Bytes.toBytes("row4"));
882 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
883 t.put(p);
884 admin.flush(tableName);
885 }
886
887
888
889
890
891 @Test(timeout = 60000)
892 public void testSplitRegionWithNoStoreFiles()
893 throws Exception {
894 final TableName tableName =
895 TableName.valueOf("testSplitRegionWithNoStoreFiles");
896
897 createTableAndWait(tableName.getName(), HConstants.CATALOG_FAMILY);
898 List<HRegion> regions = cluster.getRegions(tableName);
899 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
900 ensureTableRegionNotOnSameServerAsMeta(admin, hri);
901 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
902 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
903
904 this.admin.setBalancerRunning(false, true);
905
906 cluster.getMaster().setCatalogJanitorEnabled(false);
907 try {
908
909 printOutRegions(regionServer, "Initial regions: ");
910 Configuration conf = cluster.getConfiguration();
911 HBaseFsck.debugLsr(conf, new Path("/"));
912 Path rootDir = FSUtils.getRootDir(conf);
913 FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
914 Map<String, Path> storefiles =
915 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
916 assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
917
918
919 regions = cluster.getRegions(tableName);
920 final HRegion region = findSplittableRegion(regions);
921 assertTrue("not able to find a splittable region", region != null);
922
923
924 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
925 try {
926 st.prepare();
927 st.execute(regionServer, regionServer);
928 } catch (IOException e) {
929 fail("Split execution should have succeeded with no exceptions thrown");
930 }
931
932
933
934 List<HRegion> daughters = cluster.getRegions(tableName);
935 assertTrue(daughters.size() == 2);
936
937
938 HBaseFsck.debugLsr(conf, new Path("/"));
939 Map<String, Path> storefilesAfter =
940 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
941 assertEquals("Expected nothing but found " + storefilesAfter.toString(),
942 storefilesAfter.size(), 0);
943
944 hri = region.getRegionInfo();
945 AssignmentManager am = cluster.getMaster().getAssignmentManager();
946 RegionStates regionStates = am.getRegionStates();
947 long start = EnvironmentEdgeManager.currentTimeMillis();
948 while (!regionStates.isRegionInState(hri, State.SPLIT)) {
949 assertFalse("Timed out in waiting split parent to be in state SPLIT",
950 EnvironmentEdgeManager.currentTimeMillis() - start > 60000);
951 Thread.sleep(500);
952 }
953
954
955 am.assign(hri, true, true);
956 assertFalse("Split region can't be assigned",
957 regionStates.isRegionInTransition(hri));
958 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
959
960
961 am.unassign(hri, true, null);
962 assertFalse("Split region can't be unassigned",
963 regionStates.isRegionInTransition(hri));
964 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
965 } finally {
966 admin.setBalancerRunning(true, false);
967 cluster.getMaster().setCatalogJanitorEnabled(true);
968 }
969 }
970
971 @Test(timeout = 180000)
972 public void testSplitHooksBeforeAndAfterPONR() throws Exception {
973 TableName firstTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1");
974 TableName secondTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2");
975 HColumnDescriptor hcd = new HColumnDescriptor("cf");
976
977 HTableDescriptor desc = new HTableDescriptor(firstTable);
978 desc.addCoprocessor(MockedRegionObserver.class.getName());
979 desc.addFamily(hcd);
980 admin.createTable(desc);
981 TESTING_UTIL.waitUntilAllRegionsAssigned(firstTable);
982
983 desc = new HTableDescriptor(secondTable);
984 desc.addFamily(hcd);
985 admin.createTable(desc);
986 TESTING_UTIL.waitUntilAllRegionsAssigned(secondTable);
987
988 List<HRegion> firstTableRegions = cluster.getRegions(firstTable);
989 List<HRegion> secondTableRegions = cluster.getRegions(secondTable);
990
991
992 if (firstTableRegions.size() == 0 || secondTableRegions.size() == 0) {
993 fail("Each table should have at least one region.");
994 }
995 ServerName serverName =
996 cluster.getServerHoldingRegion(firstTableRegions.get(0).getRegionName());
997 admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
998 Bytes.toBytes(serverName.getServerName()));
999 HTable table1 = null;
1000 HTable table2 = null;
1001 try {
1002 table1 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1003 table2 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1004 insertData(firstTable.getName(), admin, table1);
1005 insertData(secondTable.getName(), admin, table2);
1006 admin.split(firstTable.getName(), "row2".getBytes());
1007 firstTableRegions = cluster.getRegions(firstTable.getName());
1008 while (firstTableRegions.size() != 2) {
1009 Thread.sleep(1000);
1010 firstTableRegions = cluster.getRegions(firstTable.getName());
1011 }
1012 assertEquals("Number of regions after split should be 2.", 2, firstTableRegions.size());
1013 secondTableRegions = cluster.getRegions(secondTable.getName());
1014 assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
1015 } finally {
1016 if (table1 != null) {
1017 table1.close();
1018 }
1019 if (table2 != null) {
1020 table2.close();
1021 }
1022 TESTING_UTIL.deleteTable(firstTable);
1023 TESTING_UTIL.deleteTable(secondTable);
1024 }
1025 }
1026
1027 private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
1028 final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
1029 try {
1030
1031 createTableAndWait(tableName, Bytes.toBytes("cf"));
1032
1033 List<HRegion> regions = awaitTableRegions(tableName);
1034 assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
1035
1036 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
1037 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1038 final HRegion region = findSplittableRegion(regions);
1039 assertTrue("not able to find a splittable region", region != null);
1040 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
1041 @Override
1042 public PairOfSameType<HRegion> stepsBeforePONR(final Server server,
1043 final RegionServerServices services, boolean testing) throws IOException {
1044 throw new SplittingNodeCreationFailedException ();
1045 }
1046 };
1047 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1048 region.getRegionInfo().getEncodedName());
1049 regionServer.getZooKeeper().sync(node);
1050 for (int i = 0; i < 100; i++) {
1051
1052
1053
1054 if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
1055 Thread.sleep(100);
1056 }
1057 }
1058 try {
1059 st.prepare();
1060 st.execute(regionServer, regionServer);
1061 } catch (IOException e) {
1062
1063
1064
1065 assertTrue("Should be instance of CreateSplittingNodeFailedException",
1066 e instanceof SplittingNodeCreationFailedException );
1067 node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1068 region.getRegionInfo().getEncodedName());
1069 {
1070 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1071 }
1072 assertTrue(st.rollback(regionServer, regionServer));
1073 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1074 }
1075 } finally {
1076 TESTING_UTIL.deleteTable(tableName);
1077 }
1078 }
1079
1080 @Test
1081 public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
1082 throws Exception {
1083 final TableName tableName =
1084 TableName.valueOf("testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck");
1085 try {
1086 HTableDescriptor htd = new HTableDescriptor(tableName);
1087 htd.addFamily(new HColumnDescriptor("f"));
1088 htd.setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName());
1089 admin.createTable(htd);
1090 List<HRegion> regions = awaitTableRegions(tableName.toBytes());
1091 HRegion region = regions.get(0);
1092 for(int i = 3;i<9;i++) {
1093 Put p = new Put(Bytes.toBytes("row"+i));
1094 p.add(Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("value"+i));
1095 region.put(p);
1096 }
1097 region.flushcache();
1098 Store store = region.getStore(Bytes.toBytes("f"));
1099 Collection<StoreFile> storefiles = store.getStorefiles();
1100 assertEquals(storefiles.size(), 1);
1101 assertFalse(region.hasReferences());
1102 Path referencePath = region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
1103 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
1104 assertNotNull(referencePath);
1105 } finally {
1106 TESTING_UTIL.deleteTable(tableName);
1107 }
1108 }
1109
1110 public static class MockedSplitTransaction extends SplitTransaction {
1111
1112 private HRegion currentRegion;
1113 public MockedSplitTransaction(HRegion r, byte[] splitrow) {
1114 super(r, splitrow);
1115 this.currentRegion = r;
1116 }
1117
1118 @Override
1119 void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b)
1120 throws IOException {
1121 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1122 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1123 try {
1124 if (!secondSplit){
1125 callRollBack = true;
1126 latch.await();
1127 }
1128 } catch (InterruptedException e) {
1129 }
1130
1131 }
1132 super.transitionZKNode(server, services, a, b);
1133 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1134 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1135 firstSplitCompleted = true;
1136 }
1137 }
1138 @Override
1139 public boolean rollback(Server server, RegionServerServices services) throws IOException {
1140 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1141 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1142 if(secondSplit){
1143 super.rollback(server, services);
1144 latch.countDown();
1145 return true;
1146 }
1147 }
1148 return super.rollback(server, services);
1149 }
1150
1151 }
1152
1153 private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
1154 for (int i = 0; i < 5; ++i) {
1155 for (HRegion r: regions) {
1156 if (r.isSplittable()) {
1157 return(r);
1158 }
1159 }
1160 Thread.sleep(100);
1161 }
1162 return(null);
1163 }
1164
1165 @Test
1166 public void testFailedSplit() throws Exception {
1167 TableName tableName = TableName.valueOf("testFailedSplit");
1168 byte[] colFamily = Bytes.toBytes("info");
1169 TESTING_UTIL.createTable(tableName, colFamily);
1170 HTable table = new HTable(TESTING_UTIL.getConfiguration(), tableName);
1171 try {
1172 TESTING_UTIL.loadTable(table, colFamily);
1173 List<HRegionInfo> regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1174 assertTrue(regions.size() == 1);
1175 final HRegion actualRegion = cluster.getRegions(tableName).get(0);
1176 actualRegion.getCoprocessorHost().load(FailingSplitRegionObserver.class,
1177 Coprocessor.PRIORITY_USER, actualRegion.getBaseConf());
1178
1179
1180 admin.split(tableName.getNameAsString());
1181 FailingSplitRegionObserver.latch.await();
1182 LOG.info("Waiting for region to come out of RIT");
1183 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1184 @Override
1185 public boolean evaluate() throws Exception {
1186 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
1187 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
1188 return !rit.containsKey(actualRegion.getRegionInfo().getEncodedName());
1189 }
1190 });
1191 regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1192 assertTrue(regions.size() == 1);
1193 } finally {
1194 table.close();
1195 TESTING_UTIL.deleteTable(tableName);
1196 }
1197 }
1198
1199 private List<HRegion> checkAndGetDaughters(byte[] tableName)
1200 throws InterruptedException {
1201 List<HRegion> daughters = null;
1202
1203 for (int i=0; i<100; i++) {
1204 daughters = cluster.getRegions(tableName);
1205 if (daughters.size() >= 2) break;
1206 Thread.sleep(100);
1207 }
1208 assertTrue(daughters.size() >= 2);
1209 return daughters;
1210 }
1211
1212 private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
1213 throws IOException, InterruptedException {
1214 cluster.abortMaster(0);
1215 cluster.waitOnMaster(0);
1216 cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
1217 MockMasterWithoutCatalogJanitor.class, HMaster.class);
1218 MockMasterWithoutCatalogJanitor master = null;
1219 master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
1220 cluster.waitForActiveAndReadyMaster();
1221 return master;
1222 }
1223
1224 private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
1225 throws IOException, InterruptedException {
1226 this.admin.split(hri.getRegionNameAsString());
1227 try {
1228 for (int i = 0; ProtobufUtil.getOnlineRegions(server).size() <= regionCount && i < 300; i++) {
1229 LOG.debug("Waiting on region to split");
1230 Thread.sleep(100);
1231 }
1232
1233 assertFalse("Waited too long for split",
1234 ProtobufUtil.getOnlineRegions(server).size() <= regionCount);
1235 } catch (RegionServerStoppedException e) {
1236 if (useZKForAssignment) {
1237
1238 LOG.error(e);
1239 throw e;
1240 }
1241 }
1242 }
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255 private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
1256 final HRegionInfo hri)
1257 throws HBaseIOException, MasterNotRunningException,
1258 ZooKeeperConnectionException, InterruptedException {
1259
1260
1261
1262 int metaServerIndex = cluster.getServerWithMeta();
1263 assertTrue(metaServerIndex != -1);
1264 HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1265 int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1266 assertTrue(tableRegionIndex != -1);
1267 HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1268 if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1269 HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1270 assertNotNull(hrs);
1271 assertNotNull(hri);
1272 LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1273 metaRegionServer.getServerName() + " to " +
1274 hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1275 admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1276 }
1277
1278 for (int i = 0; i < 100; i++) {
1279 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1280 if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1281 LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1282 tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1283 Thread.sleep(100);
1284 }
1285 assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1286 && tableRegionIndex != metaServerIndex);
1287
1288 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1289 assertTrue(tableRegionIndex != -1);
1290 assertNotSame(metaServerIndex, tableRegionIndex);
1291 return tableRegionIndex;
1292 }
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303 private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1304 final HRegionServer notThisOne) {
1305 for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1306 HRegionServer hrs = rst.getRegionServer();
1307 if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1308 if (hrs.isStopping() || hrs.isStopped()) continue;
1309 return hrs;
1310 }
1311 return null;
1312 }
1313
1314 private void printOutRegions(final HRegionServer hrs, final String prefix)
1315 throws IOException {
1316 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
1317 for (HRegionInfo region: regions) {
1318 LOG.info(prefix + region.getRegionNameAsString());
1319 }
1320 }
1321
1322 private void waitUntilRegionServerDead() throws InterruptedException {
1323
1324 for (int i=0; cluster.getMaster().getClusterStatus().
1325 getServers().size() == NB_SERVERS && i<100; i++) {
1326 LOG.info("Waiting on server to go down");
1327 Thread.sleep(100);
1328 }
1329 assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1330 getServers().size() == NB_SERVERS);
1331 }
1332
1333 private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
1334
1335 for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1336 LOG.info("Waiting for repair to happen");
1337 Thread.sleep(1000);
1338 }
1339 if (cluster.getRegions(tableName).size() < numDaughters) {
1340 fail("Waiting too long for daughter regions");
1341 }
1342 }
1343
1344 private List<HRegion> awaitTableRegions(final byte[] tableName) throws InterruptedException {
1345 List<HRegion> regions = null;
1346 for (int i = 0; i < 100; i++) {
1347 regions = cluster.getRegions(tableName);
1348 if (regions.size() > 0) break;
1349 Thread.sleep(100);
1350 }
1351 return regions;
1352 }
1353
1354 private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException,
1355 InterruptedException {
1356 HTable t = TESTING_UTIL.createTable(tableName, cf);
1357 awaitTableRegions(tableName);
1358 assertTrue("Table not online: " + Bytes.toString(tableName),
1359 cluster.getRegions(tableName).size() != 0);
1360 return t;
1361 }
1362
1363 public static class MockMasterWithoutCatalogJanitor extends HMaster {
1364
1365 public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException,
1366 InterruptedException {
1367 super(conf);
1368 }
1369 }
1370
1371 private static class SplittingNodeCreationFailedException extends IOException {
1372 private static final long serialVersionUID = 1652404976265623004L;
1373
1374 public SplittingNodeCreationFailedException () {
1375 super();
1376 }
1377 }
1378
1379 public static class MockedRegionObserver extends BaseRegionObserver {
1380 private SplitTransaction st = null;
1381 private PairOfSameType<HRegion> daughterRegions = null;
1382
1383 @Override
1384 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
1385 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
1386 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1387 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1388 List<HRegion> onlineRegions =
1389 rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
1390 HRegion region = onlineRegions.get(0);
1391 for (HRegion r : onlineRegions) {
1392 if (r.getRegionInfo().containsRow(splitKey)) {
1393 region = r;
1394 break;
1395 }
1396 }
1397 st = new SplitTransaction(region, splitKey);
1398 if (!st.prepare()) {
1399 LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
1400 + " failed. So returning null. ");
1401 ctx.bypass();
1402 return;
1403 }
1404 region.forceSplit(splitKey);
1405 daughterRegions = st.stepsBeforePONR(rs, rs, false);
1406 HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
1407 copyOfParent.setOffline(true);
1408 copyOfParent.setSplit(true);
1409
1410 Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
1411 MetaEditor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
1412 daughterRegions.getSecond().getRegionInfo());
1413 metaEntries.add(putParent);
1414
1415 Put putA = MetaEditor.makePutFromRegionInfo(daughterRegions.getFirst().getRegionInfo());
1416 Put putB = MetaEditor.makePutFromRegionInfo(daughterRegions.getSecond().getRegionInfo());
1417 st.addLocation(putA, rs.getServerName(), 1);
1418 st.addLocation(putB, rs.getServerName(), 1);
1419 metaEntries.add(putA);
1420 metaEntries.add(putB);
1421 }
1422
1423 @Override
1424 public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
1425 throws IOException {
1426 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1427 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1428 st.stepsAfterPONR(rs, rs, daughterRegions);
1429 }
1430
1431 }
1432
1433 static class CustomSplitPolicy extends RegionSplitPolicy {
1434
1435 @Override
1436 protected boolean shouldSplit() {
1437 return true;
1438 }
1439
1440 @Override
1441 public boolean skipStoreFileRangeCheck() {
1442 return true;
1443 }
1444 }
1445 }
1446