View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNotSame;
25  import static org.junit.Assert.assertNull;
26  import static org.junit.Assert.assertTrue;
27  import static org.junit.Assert.fail;
28  
29  import java.io.IOException;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.concurrent.CountDownLatch;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.Abortable;
38  import org.apache.hadoop.hbase.HBaseIOException;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.LargeTests;
45  import org.apache.hadoop.hbase.MasterNotRunningException;
46  import org.apache.hadoop.hbase.MiniHBaseCluster;
47  import org.apache.hadoop.hbase.RegionTransition;
48  import org.apache.hadoop.hbase.Server;
49  import org.apache.hadoop.hbase.ServerName;
50  import org.apache.hadoop.hbase.TableName;
51  import org.apache.hadoop.hbase.UnknownRegionException;
52  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
53  import org.apache.hadoop.hbase.catalog.MetaReader;
54  import org.apache.hadoop.hbase.client.Delete;
55  import org.apache.hadoop.hbase.client.HBaseAdmin;
56  import org.apache.hadoop.hbase.client.HTable;
57  import org.apache.hadoop.hbase.client.Put;
58  import org.apache.hadoop.hbase.client.Result;
59  import org.apache.hadoop.hbase.client.ResultScanner;
60  import org.apache.hadoop.hbase.client.Scan;
61  import org.apache.hadoop.hbase.exceptions.DeserializationException;
62  import org.apache.hadoop.hbase.executor.EventType;
63  import org.apache.hadoop.hbase.master.AssignmentManager;
64  import org.apache.hadoop.hbase.master.HMaster;
65  import org.apache.hadoop.hbase.master.RegionState;
66  import org.apache.hadoop.hbase.master.RegionState.State;
67  import org.apache.hadoop.hbase.master.RegionStates;
68  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
69  import org.apache.hadoop.hbase.util.Bytes;
70  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
71  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
72  import org.apache.hadoop.hbase.util.PairOfSameType;
73  import org.apache.hadoop.hbase.util.Threads;
74  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
75  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
76  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
77  import org.apache.zookeeper.KeeperException;
78  import org.apache.zookeeper.KeeperException.NodeExistsException;
79  import org.apache.zookeeper.data.Stat;
80  import org.junit.After;
81  import org.junit.AfterClass;
82  import org.junit.Assert;
83  import org.junit.Before;
84  import org.junit.BeforeClass;
85  import org.junit.Test;
86  import org.junit.experimental.categories.Category;
87  
88  import com.google.protobuf.ServiceException;
89  
90  /**
91   * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction}
92   * only the below tests are against a running cluster where {@link TestSplitTransaction}
93   * is tests against a bare {@link HRegion}.
94   */
95  @Category(LargeTests.class)
96  public class TestSplitTransactionOnCluster {
97    private static final Log LOG =
98      LogFactory.getLog(TestSplitTransactionOnCluster.class);
99    private HBaseAdmin admin = null;
100   private MiniHBaseCluster cluster = null;
101   private static final int NB_SERVERS = 3;
102   private static CountDownLatch latch = new CountDownLatch(1);
103   private static volatile boolean secondSplit = false;
104   private static volatile boolean callRollBack = false;
105   private static volatile boolean firstSplitCompleted = false;
106   private static byte [] CF = Bytes.toBytes("cf");
107 
108   private static final HBaseTestingUtility TESTING_UTIL =
109     new HBaseTestingUtility();
110 
111   @BeforeClass public static void before() throws Exception {
112     TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
113     // Needed because some tests have splits happening on RS that are killed
114     // We don't want to wait 3min for the master to figure it out
115     TESTING_UTIL.getConfiguration().setInt(
116         "hbase.master.assignment.timeoutmonitor.timeout", 4000);
117     TESTING_UTIL.startMiniCluster(NB_SERVERS);
118   }
119 
120   @AfterClass public static void after() throws Exception {
121     TESTING_UTIL.shutdownMiniCluster();
122   }
123 
124   @Before public void setup() throws IOException {
125     TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
126     this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
127     this.cluster = TESTING_UTIL.getMiniHBaseCluster();
128   }
129 
130   @After
131   public void tearDown() throws Exception {
132     this.admin.close();
133   }
134 
135   private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
136     assertEquals(1, regions.size());
137     HRegionInfo hri = regions.get(0).getRegionInfo();
138     return waitOnRIT(hri);
139   }
140 
141   /**
142    * Often region has not yet fully opened.  If we try to use it -- do a move for instance -- it
143    * will fail silently if the region is not yet opened.
144    * @param hri Region to check if in Regions In Transition... wait until out of transition before
145    * returning
146    * @return Passed in <code>hri</code>
147    */
148   private HRegionInfo waitOnRIT(final HRegionInfo hri) {
149     // Close worked but we are going to open the region elsewhere.  Before going on, make sure
150     // this completes.
151     while (TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
152         getRegionStates().isRegionInTransition(hri)) {
153       LOG.info("Waiting on region in transition: " +
154         TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().
155           getRegionTransitionState(hri));
156       Threads.sleep(10);
157     }
158     return hri;
159   }
160 
161   @SuppressWarnings("deprecation")
162   @Test(timeout = 60000)
163   public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
164     final TableName tableName =
165         TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
166     try {
167       // Create table then get the single region for our new table.
168       HTable t = createTableAndWait(tableName.getName(), CF);
169       final List<HRegion> regions = cluster.getRegions(tableName);
170       HRegionInfo hri = getAndCheckSingleTableRegion(regions);
171       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
172       final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
173       insertData(tableName.getName(), admin, t);
174       t.close();
175 
176       // Turn off balancer so it doesn't cut in and mess up our placements.
177       this.admin.setBalancerRunning(false, true);
178       // Turn off the meta scanner so it don't remove parent on us.
179       cluster.getMaster().setCatalogJanitorEnabled(false);
180 
181       // find a splittable region
182       final HRegion region = findSplittableRegion(regions);
183       assertTrue("not able to find a splittable region", region != null);
184 
185       new Thread() {
186         public void run() {
187           SplitTransaction st = null;
188           st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
189           try {
190             st.prepare();
191             st.execute(regionServer, regionServer);
192           } catch (IOException e) {
193 
194           }
195         }
196       }.start();
197       for (int i = 0; !callRollBack && i < 100; i++) {
198         Thread.sleep(100);
199       }
200       assertTrue("Waited too long for rollback", callRollBack);
201       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
202       try {
203         secondSplit = true;
204         // make region splittable
205         region.initialize();
206         st.prepare();
207         st.execute(regionServer, regionServer);
208       } catch (IOException e) {
209         LOG.debug("Rollback started :"+ e.getMessage());
210         st.rollback(regionServer, regionServer);
211       }
212       for (int i=0; !firstSplitCompleted && i<100; i++) {
213         Thread.sleep(100);
214       }
215       assertTrue("fist split did not complete", firstSplitCompleted);
216 
217       RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
218       Map<String, RegionState> rit = regionStates.getRegionsInTransition();
219 
220       for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
221         Thread.sleep(100);
222       }
223       assertFalse("region still in transition", rit.containsKey(
224           rit.containsKey(hri.getTable())));
225 
226       List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
227       // Region server side split is successful.
228       assertEquals("The parent region should be splitted", 2, onlineRegions.size());
229       //Should be present in RIT
230       List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
231           .getRegionStates().getRegionsOfTable(tableName);
232       // Master side should also reflect the same
233       assertEquals("No of regions in master", 2, regionsOfTable.size());
234     } finally {
235       admin.setBalancerRunning(true, false);
236       secondSplit = false;
237       firstSplitCompleted = false;
238       callRollBack = false;
239       cluster.getMaster().setCatalogJanitorEnabled(true);
240       TESTING_UTIL.deleteTable(tableName);
241     }
242   }
243 
244  /**
245    * A test that intentionally has master fail the processing of the split message.
246    * Tests that the regionserver split ephemeral node gets cleaned up if it
247    * crashes and that after we process server shutdown, the daughters are up on
248    * line.
249    * @throws IOException
250    * @throws InterruptedException
251    * @throws NodeExistsException
252    * @throws KeeperException
253    * @throws DeserializationException
254    */
255   @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
256   throws IOException, InterruptedException, NodeExistsException, KeeperException,
257       DeserializationException, ServiceException {
258     final byte [] tableName =
259       Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
260 
261     // Create table then get the single region for our new table.
262     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
263     List<HRegion> regions = cluster.getRegions(tableName);
264     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
265 
266     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
267 
268     // Turn off balancer so it doesn't cut in and mess up our placements.
269     this.admin.setBalancerRunning(false, true);
270     // Turn off the meta scanner so it don't remove parent on us.
271     cluster.getMaster().setCatalogJanitorEnabled(false);
272     try {
273       // Add a bit of load up into the table so splittable.
274       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
275       // Get region pre-split.
276       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
277       printOutRegions(server, "Initial regions: ");
278       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
279       // Now, before we split, set special flag in master, a flag that has
280       // it FAIL the processing of split.
281       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
282       // Now try splitting and it should work.
283       split(hri, server, regionCount);
284       // Get daughters
285       List<HRegion> daughters = checkAndGetDaughters(tableName);
286       // Assert the ephemeral node is up in zk.
287       String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
288         hri.getEncodedName());
289       RegionTransition rt = null;
290       Stat stats = null;
291       // Wait till the znode moved to SPLIT
292       for (int i=0; i<100; i++) {
293         stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
294         rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
295           hri.getEncodedName()));
296         if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
297         Thread.sleep(100);
298       }
299       LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
300       assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
301       // Now crash the server
302       cluster.abortRegionServer(tableRegionIndex);
303       waitUntilRegionServerDead();
304       awaitDaughters(tableName, daughters.size());
305 
306       // Assert daughters are online.
307       regions = cluster.getRegions(tableName);
308       for (HRegion r: regions) {
309         assertTrue(daughters.contains(r));
310       }
311       // Finally assert that the ephemeral SPLIT znode was cleaned up.
312       for (int i=0; i<100; i++) {
313         // wait a bit (10s max) for the node to disappear
314         stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
315         if (stats == null) break;
316         Thread.sleep(100);
317       }
318       LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
319       assertTrue(stats == null);
320     } finally {
321       // Set this flag back.
322       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
323       admin.setBalancerRunning(true, false);
324       cluster.getMaster().setCatalogJanitorEnabled(true);
325       t.close();
326     }
327   }
328 
329   @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
330   throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
331     final byte [] tableName =
332       Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
333 
334     // Create table then get the single region for our new table.
335     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
336     List<HRegion> regions = cluster.getRegions(tableName);
337     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
338 
339     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
340 
341     // Turn off balancer so it doesn't cut in and mess up our placements.
342     this.admin.setBalancerRunning(false, true);
343     // Turn off the meta scanner so it don't remove parent on us.
344     cluster.getMaster().setCatalogJanitorEnabled(false);
345     try {
346       // Add a bit of load up into the table so splittable.
347       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
348       // Get region pre-split.
349       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
350       printOutRegions(server, "Initial regions: ");
351       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
352       // Insert into zk a blocking znode, a znode of same name as region
353       // so it gets in way of our splitting.
354       ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
355       ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
356         hri, fakedServer);
357       // Now try splitting.... should fail.  And each should successfully
358       // rollback.
359       this.admin.split(hri.getRegionNameAsString());
360       this.admin.split(hri.getRegionNameAsString());
361       this.admin.split(hri.getRegionNameAsString());
362       // Wait around a while and assert count of regions remains constant.
363       for (int i = 0; i < 10; i++) {
364         Thread.sleep(100);
365         assertEquals(regionCount, ProtobufUtil.getOnlineRegions(server).size());
366       }
367       // Now clear the zknode
368       ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
369         hri, fakedServer);
370       // Now try splitting and it should work.
371       split(hri, server, regionCount);
372       // Get daughters
373       checkAndGetDaughters(tableName);
374       // OK, so split happened after we cleared the blocking node.
375     } finally {
376       admin.setBalancerRunning(true, false);
377       cluster.getMaster().setCatalogJanitorEnabled(true);
378       t.close();
379     }
380   }
381 
382   /**
383    * Test that if daughter split on us, we won't do the shutdown handler fixup
384    * just because we can't find the immediate daughter of an offlined parent.
385    * @throws IOException
386    * @throws InterruptedException
387    */
388   @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
389   throws IOException, InterruptedException, ServiceException {
390     final byte [] tableName =
391       Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
392 
393     // Create table then get the single region for our new table.
394     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
395     List<HRegion> regions = cluster.getRegions(tableName);
396     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
397 
398     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
399 
400     // Turn off balancer so it doesn't cut in and mess up our placements.
401     this.admin.setBalancerRunning(false, true);
402     // Turn off the meta scanner so it don't remove parent on us.
403     cluster.getMaster().setCatalogJanitorEnabled(false);
404     try {
405       // Add a bit of load up into the table so splittable.
406       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
407       // Get region pre-split.
408       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
409       printOutRegions(server, "Initial regions: ");
410       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
411       // Now split.
412       split(hri, server, regionCount);
413       // Get daughters
414       List<HRegion> daughters = checkAndGetDaughters(tableName);
415       // Now split one of the daughters.
416       regionCount = ProtobufUtil.getOnlineRegions(server).size();
417       HRegionInfo daughter = daughters.get(0).getRegionInfo();
418       LOG.info("Daughter we are going to split: " + daughter);
419       // Compact first to ensure we have cleaned up references -- else the split
420       // will fail.
421       this.admin.compact(daughter.getRegionName());
422       daughters = cluster.getRegions(tableName);
423       HRegion daughterRegion = null;
424       for (HRegion r: daughters) {
425         if (r.getRegionInfo().equals(daughter)) {
426           daughterRegion = r;
427           LOG.info("Found matching HRI: " + daughterRegion);
428           break;
429         }
430       }
431       assertTrue(daughterRegion != null);
432       for (int i=0; i<100; i++) {
433         if (!daughterRegion.hasReferences()) break;
434         Threads.sleep(100);
435       }
436       assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
437       LOG.info("Daughter hri before split (has been compacted): " + daughter);
438       split(daughter, server, regionCount);
439       // Get list of daughters
440       daughters = cluster.getRegions(tableName);
441       for (HRegion d: daughters) {
442         LOG.info("Regions before crash: " + d);
443       }
444       // Now crash the server
445       cluster.abortRegionServer(tableRegionIndex);
446       waitUntilRegionServerDead();
447       awaitDaughters(tableName, daughters.size());
448       // Assert daughters are online and ONLY the original daughters -- that
449       // fixup didn't insert one during server shutdown recover.
450       regions = cluster.getRegions(tableName);
451       for (HRegion d: daughters) {
452         LOG.info("Regions after crash: " + d);
453       }
454       assertEquals(daughters.size(), regions.size());
455       for (HRegion r: regions) {
456         LOG.info("Regions post crash " + r);
457         assertTrue("Missing region post crash " + r, daughters.contains(r));
458       }
459     } finally {
460       admin.setBalancerRunning(true, false);
461       cluster.getMaster().setCatalogJanitorEnabled(true);
462       t.close();
463     }
464   }
465 
466   @Test(timeout = 180000)
467   public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
468     Configuration conf = TESTING_UTIL.getConfiguration();
469     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
470     TableName userTableName =
471         TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
472     HTableDescriptor htd = new HTableDescriptor(userTableName);
473     HColumnDescriptor hcd = new HColumnDescriptor("col");
474     htd.addFamily(hcd);
475     admin.createTable(htd);
476     ZKAssign.blockUntilNoRIT(zkw);
477     HTable table = new HTable(conf, userTableName);
478     try {
479       for (int i = 0; i <= 5; i++) {
480         String row = "row" + i;
481         Put p = new Put(row.getBytes());
482         String val = "Val" + i;
483         p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
484         table.put(p);
485         admin.flush(userTableName.getName());
486         Delete d = new Delete(row.getBytes());
487         // Do a normal delete
488         table.delete(d);
489         admin.flush(userTableName.getName());
490       }
491       admin.majorCompact(userTableName.getName());
492       List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
493           .getMaster().getAssignmentManager().getRegionStates()
494           .getRegionsOfTable(userTableName);
495       HRegionInfo hRegionInfo = regionsOfTable.get(0);
496       Put p = new Put("row6".getBytes());
497       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
498       table.put(p);
499       p = new Put("row7".getBytes());
500       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
501       table.put(p);
502       p = new Put("row8".getBytes());
503       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
504       table.put(p);
505       admin.flush(userTableName.getName());
506       admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
507       regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
508           .getAssignmentManager().getRegionStates()
509           .getRegionsOfTable(userTableName);
510 
511       while (regionsOfTable.size() != 2) {
512         Thread.sleep(2000);
513         regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
514             .getAssignmentManager().getRegionStates()
515             .getRegionsOfTable(userTableName);
516       }
517       Assert.assertEquals(2, regionsOfTable.size());
518       Scan s = new Scan();
519       ResultScanner scanner = table.getScanner(s);
520       int mainTableCount = 0;
521       for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
522         mainTableCount++;
523       }
524       Assert.assertEquals(3, mainTableCount);
525     } finally {
526       table.close();
527     }
528   }
529 
530   /**
531    * Noop Abortable implementation used below in tests.
532    */
533   static class UselessTestAbortable implements Abortable {
534     boolean aborted = false;
535     @Override
536     public void abort(String why, Throwable e) {
537       LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
538       aborted = true;
539     }
540 
541     @Override
542     public boolean isAborted() {
543       return this.aborted;
544     }
545   }
546 
547   /**
548    * Verifies HBASE-5806.  When splitting is partially done and the master goes down
549    * when the SPLIT node is in either SPLIT or SPLITTING state.
550    *
551    * @throws IOException
552    * @throws InterruptedException
553    * @throws NodeExistsException
554    * @throws KeeperException
555    * @throws DeserializationException
556    */
557   @Test(timeout = 400000)
558   public void testMasterRestartWhenSplittingIsPartial()
559       throws IOException, InterruptedException, NodeExistsException,
560       KeeperException, DeserializationException, ServiceException {
561     final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
562 
563     // Create table then get the single region for our new table.
564     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
565     List<HRegion> regions = cluster.getRegions(tableName);
566     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
567 
568     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
569 
570     // Turn off balancer so it doesn't cut in and mess up our placements.
571     this.admin.setBalancerRunning(false, true);
572     // Turn off the meta scanner so it don't remove parent on us.
573     cluster.getMaster().setCatalogJanitorEnabled(false);
574     ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
575       "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
576     try {
577       // Add a bit of load up into the table so splittable.
578       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
579       // Get region pre-split.
580       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
581       printOutRegions(server, "Initial regions: ");
582       // Now, before we split, set special flag in master, a flag that has
583       // it FAIL the processing of split.
584       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
585       // Now try splitting and it should work.
586 
587       this.admin.split(hri.getRegionNameAsString());
588       checkAndGetDaughters(tableName);
589       // Assert the ephemeral node is up in zk.
590       String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
591       Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
592       LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
593           + stats);
594       byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
595       RegionTransition rtd = RegionTransition.parseFrom(bytes);
596       // State could be SPLIT or SPLITTING.
597       assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
598           || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
599 
600       // abort and wait for new master.
601       MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
602 
603       this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
604 
605       // Update the region to be offline and split, so that HRegionInfo#equals
606       // returns true in checking rebuilt region states map.
607       hri.setOffline(true);
608       hri.setSplit(true);
609       ServerName regionServerOfRegion = master.getAssignmentManager()
610         .getRegionStates().getRegionServerOfRegion(hri);
611       assertTrue(regionServerOfRegion != null);
612 
613       // Remove the block so that split can move ahead.
614       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
615       String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
616       Stat stat = new Stat();
617       byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
618       // ZKUtil.create
619       for (int i=0; data != null && i<60; i++) {
620         Thread.sleep(1000);
621         data = ZKUtil.getDataNoWatch(zkw, node, stat);
622       }
623       assertNull("Waited too long for ZK node to be removed: "+node, data);
624       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
625       assertTrue("Split parent should be in SPLIT state",
626         regionStates.isRegionInState(hri, State.SPLIT));
627       regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
628       assertTrue(regionServerOfRegion == null);
629     } finally {
630       // Set this flag back.
631       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
632       admin.setBalancerRunning(true, false);
633       cluster.getMaster().setCatalogJanitorEnabled(true);
634       t.close();
635       zkw.close();
636     }
637   }
638 
639   /**
640    * Verifies HBASE-5806.  Here the case is that splitting is completed but before the
641    * CJ could remove the parent region the master is killed and restarted.
642    * @throws IOException
643    * @throws InterruptedException
644    * @throws NodeExistsException
645    * @throws KeeperException
646    */
647   @Test (timeout = 300000)
648   public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
649       throws IOException, InterruptedException, NodeExistsException,
650       KeeperException, ServiceException {
651     final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
652 
653     // Create table then get the single region for our new table.
654     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
655     List<HRegion> regions = cluster.getRegions(tableName);
656     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
657 
658     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
659 
660     // Turn off balancer so it doesn't cut in and mess up our placements.
661     this.admin.setBalancerRunning(false, true);
662     // Turn off the meta scanner so it don't remove parent on us.
663     cluster.getMaster().setCatalogJanitorEnabled(false);
664     ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
665       "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
666     try {
667       // Add a bit of load up into the table so splittable.
668       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
669       // Get region pre-split.
670       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
671       printOutRegions(server, "Initial regions: ");
672 
673       this.admin.split(hri.getRegionNameAsString());
674       checkAndGetDaughters(tableName);
675       // Assert the ephemeral node is up in zk.
676       String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
677       Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
678       LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
679           + stats);
680       String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
681       Stat stat = new Stat();
682       byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
683       // ZKUtil.create
684       for (int i=0; data != null && i<60; i++) {
685         Thread.sleep(1000);
686         data = ZKUtil.getDataNoWatch(zkw, node, stat);
687       }
688       assertNull("Waited too long for ZK node to be removed: "+node, data);
689 
690       MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
691 
692       this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
693 
694       // Update the region to be offline and split, so that HRegionInfo#equals
695       // returns true in checking rebuilt region states map.
696       hri.setOffline(true);
697       hri.setSplit(true);
698       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
699       assertTrue("Split parent should be in SPLIT state",
700         regionStates.isRegionInState(hri, State.SPLIT));
701       ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
702       assertTrue(regionServerOfRegion == null);
703     } finally {
704       this.admin.setBalancerRunning(true, false);
705       cluster.getMaster().setCatalogJanitorEnabled(true);
706       t.close();
707       zkw.close();
708     }
709   }
710 
711   /**
712    *
713    * While transitioning node from RS_ZK_REGION_SPLITTING to
714    * RS_ZK_REGION_SPLITTING during region split,if zookeper went down split always
715    * fails for the region. HBASE-6088 fixes this scenario.
716    * This test case is to test the znode is deleted(if created) or not in roll back.
717    *
718    * @throws IOException
719    * @throws InterruptedException
720    * @throws KeeperException
721    */
722   @Test
723   public void testSplitBeforeSettingSplittingInZK() throws Exception,
724       InterruptedException, KeeperException {
725     testSplitBeforeSettingSplittingInZKInternals();
726   }
727 
728   @Test(timeout = 60000)
729   public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
730     final TableName tableName =
731         TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
732     // Create table then get the single region for our new table.
733     HTable t = createTableAndWait(tableName.getName(), CF);
734     try {
735       List<HRegion> regions = cluster.getRegions(tableName);
736       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
737       HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
738       insertData(tableName.getName(), admin, t);
739       // Turn off balancer so it doesn't cut in and mess up our placements.
740       admin.setBalancerRunning(false, true);
741       // Turn off the meta scanner so it don't remove parent on us.
742       cluster.getMaster().setCatalogJanitorEnabled(false);
743       boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
744           tableName);
745       assertEquals("The specified table should present.", true, tableExists);
746       final HRegion region = findSplittableRegion(regions);
747       assertTrue("not able to find a splittable region", region != null);
748       SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
749       try {
750         st.prepare();
751         st.createDaughters(regionServer, regionServer);
752       } catch (IOException e) {
753 
754       }
755       tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
756           tableName);
757       assertEquals("The specified table should present.", true, tableExists);
758     } finally {
759       admin.setBalancerRunning(true, false);
760       cluster.getMaster().setCatalogJanitorEnabled(true);
761       t.close();
762     }
763   }
764 
765   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
766       InterruptedException {
767     Put p = new Put(Bytes.toBytes("row1"));
768     p.add(CF, Bytes.toBytes("q1"), Bytes.toBytes("1"));
769     t.put(p);
770     p = new Put(Bytes.toBytes("row2"));
771     p.add(CF, Bytes.toBytes("q1"), Bytes.toBytes("2"));
772     t.put(p);
773     p = new Put(Bytes.toBytes("row3"));
774     p.add(CF, Bytes.toBytes("q1"), Bytes.toBytes("3"));
775     t.put(p);
776     p = new Put(Bytes.toBytes("row4"));
777     p.add(CF, Bytes.toBytes("q1"), Bytes.toBytes("4"));
778     t.put(p);
779     admin.flush(tableName);
780   }
781 
782   /**
783    * After a region is split, it should not be able to assign again
784    */
785   @Test
786   public void testSplitRegionNotAssignable() throws Exception {
787     final TableName tableName =
788         TableName.valueOf("testSplitRegionWithNoStoreFiles");
789     // Create table then get the single region for our new table.
790     HTable t = createTableAndWait(tableName.getName(), CF);
791     try {
792       List<HRegion> regions = cluster.getRegions(tableName);
793       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
794       HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
795       insertData(tableName.getName(), admin, t);
796       // Turn off balancer so it doesn't cut in and mess up our placements.
797       admin.setBalancerRunning(false, true);
798       // Turn off the meta scanner so it don't remove parent on us.
799       cluster.getMaster().setCatalogJanitorEnabled(false);
800       final HRegion region = findSplittableRegion(regions);
801       assertTrue("not able to find a splittable region", region != null);
802 
803       // Now split.
804       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
805       try {
806         st.prepare();
807         st.execute(regionServer, regionServer);
808       } catch (IOException e) {
809         fail("Split execution should have succeeded with no exceptions thrown");
810       }
811 
812       List<HRegion> daughters = cluster.getRegions(tableName);
813       assertTrue(daughters.size() == regions.size() + 1);
814 
815       HRegionInfo hri = region.getRegionInfo(); // split parent
816       AssignmentManager am = cluster.getMaster().getAssignmentManager();
817       RegionStates regionStates = am.getRegionStates();
818       long start = EnvironmentEdgeManager.currentTimeMillis();
819       while (!regionStates.isRegionInState(hri, State.SPLIT)) {
820         assertFalse("Timed out in waiting split parent to be in state SPLIT",
821           EnvironmentEdgeManager.currentTimeMillis() - start > 60000);
822         Thread.sleep(500);
823       }
824 
825       // We should not be able to assign it again
826       am.assign(hri, true, true);
827       assertFalse("Split region can't be assigned",
828         regionStates.isRegionInTransition(hri));
829       assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
830 
831       // We should not be able to unassign it either
832       am.unassign(hri, true, null);
833       assertFalse("Split region can't be unassigned",
834         regionStates.isRegionInTransition(hri));
835       assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
836     } finally {
837       admin.setBalancerRunning(true, false);
838       cluster.getMaster().setCatalogJanitorEnabled(true);
839     }
840   }
841 
842   private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
843     final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
844     try {
845       // Create table then get the single region for our new table.
846       createTableAndWait(tableName, CF);
847 
848       List<HRegion> regions = awaitTableRegions(tableName);
849       assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
850 
851       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
852       HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
853       final HRegion region = findSplittableRegion(regions);
854       assertTrue("not able to find a splittable region", region != null);
855       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
856         @Override
857         PairOfSameType<HRegion> createDaughters(final Server server,
858             final RegionServerServices services) throws IOException {
859           throw new SplittingNodeCreationFailedException ();
860         }
861       };
862       String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
863           region.getRegionInfo().getEncodedName());
864       regionServer.getZooKeeper().sync(node);
865       for (int i = 0; i < 100; i++) {
866         // We expect the znode to be deleted by this time. Here the
867         // znode could be in OPENED state and the
868         // master has not yet deleted the znode.
869         if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
870           Thread.sleep(100);
871         }
872       }
873       try {
874         st.prepare();
875         st.execute(regionServer, regionServer);
876       } catch (IOException e) {
877         // check for the specific instance in case the Split failed due to the
878         // existence of the znode in OPENED state.
879         // This will at least make the test to fail;
880         assertTrue("Should be instance of CreateSplittingNodeFailedException",
881             e instanceof SplittingNodeCreationFailedException );
882         node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
883             region.getRegionInfo().getEncodedName());
884         {
885           assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
886         }
887         assertTrue(st.rollback(regionServer, regionServer));
888         assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
889       }
890     } finally {
891       TESTING_UTIL.deleteTable(tableName);
892     }
893   }
894 
895   public static class MockedSplitTransaction extends SplitTransaction {
896 
897     private HRegion currentRegion;
898     public MockedSplitTransaction(HRegion r, byte[] splitrow) {
899       super(r, splitrow);
900       this.currentRegion = r;
901     }
902 
903     @Override
904     void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b)
905         throws IOException {
906       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
907           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
908         try {
909           if (!secondSplit){
910             callRollBack = true;
911             latch.await();
912           }
913         } catch (InterruptedException e) {
914         }
915 
916       }
917       super.transitionZKNode(server, services, a, b);
918       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
919           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
920         firstSplitCompleted = true;
921       }
922     }
923     @Override
924     public boolean rollback(Server server, RegionServerServices services) throws IOException {
925       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
926           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
927         if(secondSplit){
928           super.rollback(server, services);
929           latch.countDown();
930           return true;
931         }
932       }
933       return super.rollback(server, services);
934     }
935 
936   }
937 
938   private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
939     for (int i = 0; i < 5; ++i) {
940       for (HRegion r: regions) {
941         if (r.isSplittable()) {
942           return(r);
943         }
944       }
945       Thread.sleep(100);
946     }
947     return(null);
948   }
949 
950   private List<HRegion> checkAndGetDaughters(byte[] tableName)
951       throws InterruptedException {
952     List<HRegion> daughters = null;
953     // try up to 10s
954     for (int i=0; i<100; i++) {
955       daughters = cluster.getRegions(tableName);
956       if (daughters.size() >= 2) break;
957       Thread.sleep(100);
958     }
959     assertTrue(daughters.size() >= 2);
960     return daughters;
961   }
962 
963   private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
964   throws IOException, InterruptedException {
965     cluster.abortMaster(0);
966     cluster.waitOnMaster(0);
967     cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
968     		MockMasterWithoutCatalogJanitor.class, HMaster.class);
969     MockMasterWithoutCatalogJanitor master = null;
970     master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
971     cluster.waitForActiveAndReadyMaster();
972     return master;
973   }
974 
975   private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
976       throws IOException, InterruptedException {
977     this.admin.split(hri.getRegionNameAsString());
978     for (int i = 0; ProtobufUtil.getOnlineRegions(server).size() <= regionCount && i < 300; i++) {
979       LOG.debug("Waiting on region to split");
980       Thread.sleep(100);
981     }
982 
983     assertFalse("Waited too long for split",
984         ProtobufUtil.getOnlineRegions(server).size() <= regionCount);
985   }
986 
987   /**
988    * Ensure single table region is not on same server as the single hbase:meta table
989    * region.
990    * @param admin
991    * @param hri
992    * @return Index of the server hosting the single table region
993    * @throws UnknownRegionException
994    * @throws MasterNotRunningException
995    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
996    * @throws InterruptedException
997    */
998   private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
999       final HRegionInfo hri)
1000   throws HBaseIOException, MasterNotRunningException,
1001   ZooKeeperConnectionException, InterruptedException {
1002     // Now make sure that the table region is not on same server as that hosting
1003     // hbase:meta  We don't want hbase:meta replay polluting our test when we later crash
1004     // the table region serving server.
1005     int metaServerIndex = cluster.getServerWithMeta();
1006     assertTrue(metaServerIndex != -1);
1007     HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1008     int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1009     assertTrue(tableRegionIndex != -1);
1010     HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1011     if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1012       HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1013       assertNotNull(hrs);
1014       assertNotNull(hri);
1015       LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1016         metaRegionServer.getServerName() + " to " +
1017         hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1018       admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1019     }
1020     // Wait till table region is up on the server that is NOT carrying hbase:meta.
1021     for (int i = 0; i < 100; i++) {
1022       tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1023       if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1024       LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1025         tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1026       Thread.sleep(100);
1027     }
1028     assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1029         && tableRegionIndex != metaServerIndex);
1030     // Verify for sure table region is not on same server as hbase:meta
1031     tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1032     assertTrue(tableRegionIndex != -1);
1033     assertNotSame(metaServerIndex, tableRegionIndex);
1034     return tableRegionIndex;
1035   }
1036 
1037   /**
1038    * Find regionserver other than the one passed.
1039    * Can't rely on indexes into list of regionservers since crashed servers
1040    * occupy an index.
1041    * @param cluster
1042    * @param notThisOne
1043    * @return A regionserver that is not <code>notThisOne</code> or null if none
1044    * found
1045    */
1046   private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1047       final HRegionServer notThisOne) {
1048     for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1049       HRegionServer hrs = rst.getRegionServer();
1050       if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1051       if (hrs.isStopping() || hrs.isStopped()) continue;
1052       return hrs;
1053     }
1054     return null;
1055   }
1056 
1057   private void printOutRegions(final HRegionServer hrs, final String prefix)
1058       throws IOException {
1059     List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
1060     for (HRegionInfo region: regions) {
1061       LOG.info(prefix + region.getRegionNameAsString());
1062     }
1063   }
1064 
1065   private void waitUntilRegionServerDead() throws InterruptedException {
1066     // Wait until the master processes the RS shutdown
1067     for (int i=0; cluster.getMaster().getClusterStatus().
1068         getServers().size() == NB_SERVERS && i<100; i++) {
1069       LOG.info("Waiting on server to go down");
1070       Thread.sleep(100);
1071     }
1072     assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1073         getServers().size() == NB_SERVERS);
1074   }
1075 
1076   private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
1077     // Wait till regions are back on line again.
1078     for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1079       LOG.info("Waiting for repair to happen");
1080       Thread.sleep(1000);
1081     }
1082     if (cluster.getRegions(tableName).size() < numDaughters) {
1083       fail("Waiting too long for daughter regions");
1084     }
1085   }
1086 
1087   private List<HRegion> awaitTableRegions(final byte[] tableName) throws InterruptedException {
1088     List<HRegion> regions = null;
1089     for (int i = 0; i < 100; i++) {
1090       regions = cluster.getRegions(tableName);
1091       if (regions.size() > 0) break;
1092       Thread.sleep(100);
1093     }
1094     return regions;
1095   }
1096 
1097   private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException,
1098       InterruptedException {
1099     HTable t = TESTING_UTIL.createTable(tableName, cf);
1100     awaitTableRegions(tableName);
1101     assertTrue("Table not online: " + Bytes.toString(tableName),
1102       cluster.getRegions(tableName).size() != 0);
1103     return t;
1104   }
1105 
1106   public static class MockMasterWithoutCatalogJanitor extends HMaster {
1107 
1108     public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException,
1109         InterruptedException {
1110       super(conf);
1111     }
1112 
1113     protected void startCatalogJanitorChore() {
1114       LOG.debug("Customised master executed.");
1115     }
1116   }
1117 
1118   private static class SplittingNodeCreationFailedException  extends IOException {
1119     private static final long serialVersionUID = 1652404976265623004L;
1120 
1121     public SplittingNodeCreationFailedException () {
1122       super();
1123     }
1124   }
1125 }
1126