View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNotSame;
25  import static org.junit.Assert.assertNull;
26  import static org.junit.Assert.assertTrue;
27  import static org.junit.Assert.fail;
28  
29  import java.io.IOException;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.concurrent.CountDownLatch;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.hbase.Abortable;
40  import org.apache.hadoop.hbase.Coprocessor;
41  import org.apache.hadoop.hbase.HBaseIOException;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.LargeTests;
48  import org.apache.hadoop.hbase.MasterNotRunningException;
49  import org.apache.hadoop.hbase.MiniHBaseCluster;
50  import org.apache.hadoop.hbase.RegionTransition;
51  import org.apache.hadoop.hbase.Server;
52  import org.apache.hadoop.hbase.ServerName;
53  import org.apache.hadoop.hbase.TableName;
54  import org.apache.hadoop.hbase.UnknownRegionException;
55  import org.apache.hadoop.hbase.Waiter;
56  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
57  import org.apache.hadoop.hbase.catalog.MetaEditor;
58  import org.apache.hadoop.hbase.catalog.MetaReader;
59  import org.apache.hadoop.hbase.client.Delete;
60  import org.apache.hadoop.hbase.client.HBaseAdmin;
61  import org.apache.hadoop.hbase.client.HTable;
62  import org.apache.hadoop.hbase.client.Mutation;
63  import org.apache.hadoop.hbase.client.Put;
64  import org.apache.hadoop.hbase.client.Result;
65  import org.apache.hadoop.hbase.client.ResultScanner;
66  import org.apache.hadoop.hbase.client.Scan;
67  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
68  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
69  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
70  import org.apache.hadoop.hbase.exceptions.DeserializationException;
71  import org.apache.hadoop.hbase.executor.EventType;
72  import org.apache.hadoop.hbase.master.AssignmentManager;
73  import org.apache.hadoop.hbase.master.HMaster;
74  import org.apache.hadoop.hbase.master.RegionState;
75  import org.apache.hadoop.hbase.master.RegionStates;
76  import org.apache.hadoop.hbase.master.RegionState.State;
77  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
78  import org.apache.hadoop.hbase.util.Bytes;
79  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
80  import org.apache.hadoop.hbase.util.FSUtils;
81  import org.apache.hadoop.hbase.util.HBaseFsck;
82  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
83  import org.apache.hadoop.hbase.util.PairOfSameType;
84  import org.apache.hadoop.hbase.util.Threads;
85  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
86  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
87  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
88  import org.apache.zookeeper.KeeperException;
89  import org.apache.zookeeper.KeeperException.NodeExistsException;
90  import org.apache.zookeeper.data.Stat;
91  import org.junit.After;
92  import org.junit.AfterClass;
93  import org.junit.Assert;
94  import org.junit.Before;
95  import org.junit.BeforeClass;
96  import org.junit.Test;
97  import org.junit.experimental.categories.Category;
98  
99  import com.google.protobuf.ServiceException;
100 
101 /**
102  * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction}
103  * only the below tests are against a running cluster where {@link TestSplitTransaction}
104  * is tests against a bare {@link HRegion}.
105  */
106 @Category(LargeTests.class)
107 public class TestSplitTransactionOnCluster {
108   private static final Log LOG =
109     LogFactory.getLog(TestSplitTransactionOnCluster.class);
110   private HBaseAdmin admin = null;
111   private MiniHBaseCluster cluster = null;
112   private static final int NB_SERVERS = 3;
113   private static CountDownLatch latch = new CountDownLatch(1);
114   private static volatile boolean secondSplit = false;
115   private static volatile boolean callRollBack = false;
116   private static volatile boolean firstSplitCompleted = false;
117   private static boolean useZKForAssignment = true;
118 
119   static final HBaseTestingUtility TESTING_UTIL =
120     new HBaseTestingUtility();
121 
122   static void setupOnce() throws Exception {
123     TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
124     useZKForAssignment =
125         TESTING_UTIL.getConfiguration().getBoolean("hbase.assignment.usezk", false);
126     TESTING_UTIL.startMiniCluster(NB_SERVERS);
127   }
128 
129   @BeforeClass public static void before() throws Exception {
130     // Use ZK for region assignment
131     TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
132     setupOnce();
133   }
134 
135   @AfterClass public static void after() throws Exception {
136     TESTING_UTIL.shutdownMiniCluster();
137   }
138 
139   @Before public void setup() throws IOException {
140     TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
141     this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
142     this.cluster = TESTING_UTIL.getMiniHBaseCluster();
143   }
144 
145   @After
146   public void tearDown() throws Exception {
147     this.admin.close();
148   }
149 
150   private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
151     assertEquals(1, regions.size());
152     HRegionInfo hri = regions.get(0).getRegionInfo();
153     return waitOnRIT(hri);
154   }
155 
156   /**
157    * Often region has not yet fully opened.  If we try to use it -- do a move for instance -- it
158    * will fail silently if the region is not yet opened.
159    * @param hri Region to check if in Regions In Transition... wait until out of transition before
160    * returning
161    * @return Passed in <code>hri</code>
162    */
163   private HRegionInfo waitOnRIT(final HRegionInfo hri) {
164     // Close worked but we are going to open the region elsewhere.  Before going on, make sure
165     // this completes.
166     while (TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
167         getRegionStates().isRegionInTransition(hri)) {
168       LOG.info("Waiting on region in transition: " +
169         TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().
170           getRegionTransitionState(hri));
171       Threads.sleep(10);
172     }
173     return hri;
174   }
175 
176   @SuppressWarnings("deprecation")
177   @Test(timeout = 60000)
178   public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
179     final TableName tableName =
180         TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
181 
182     if (!useZKForAssignment) {
183       // This test doesn't apply if not using ZK for assignment
184       return;
185     }
186 
187     try {
188       // Create table then get the single region for our new table.
189       HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
190       final List<HRegion> regions = cluster.getRegions(tableName);
191       HRegionInfo hri = getAndCheckSingleTableRegion(regions);
192       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
193       final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
194       insertData(tableName.getName(), admin, t);
195       t.close();
196 
197       // Turn off balancer so it doesn't cut in and mess up our placements.
198       this.admin.setBalancerRunning(false, true);
199       // Turn off the meta scanner so it don't remove parent on us.
200       cluster.getMaster().setCatalogJanitorEnabled(false);
201 
202       // find a splittable region
203       final HRegion region = findSplittableRegion(regions);
204       assertTrue("not able to find a splittable region", region != null);
205 
206       new Thread() {
207         @Override
208         public void run() {
209           SplitTransaction st = null;
210           st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
211           try {
212             st.prepare();
213             st.execute(regionServer, regionServer);
214           } catch (IOException e) {
215 
216           }
217         }
218       }.start();
219       for (int i = 0; !callRollBack && i < 100; i++) {
220         Thread.sleep(100);
221       }
222       assertTrue("Waited too long for rollback", callRollBack);
223       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
224       try {
225         secondSplit = true;
226         // make region splittable
227         region.initialize();
228         st.prepare();
229         st.execute(regionServer, regionServer);
230       } catch (IOException e) {
231         LOG.debug("Rollback started :"+ e.getMessage());
232         st.rollback(regionServer, regionServer);
233       }
234       for (int i=0; !firstSplitCompleted && i<100; i++) {
235         Thread.sleep(100);
236       }
237       assertTrue("fist split did not complete", firstSplitCompleted);
238 
239       RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
240       Map<String, RegionState> rit = regionStates.getRegionsInTransition();
241 
242       for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
243         Thread.sleep(100);
244       }
245       assertFalse("region still in transition", rit.containsKey(
246           rit.containsKey(hri.getTable())));
247 
248       List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
249       // Region server side split is successful.
250       assertEquals("The parent region should be splitted", 2, onlineRegions.size());
251       //Should be present in RIT
252       List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
253           .getRegionStates().getRegionsOfTable(tableName);
254       // Master side should also reflect the same
255       assertEquals("No of regions in master", 2, regionsOfTable.size());
256     } finally {
257       admin.setBalancerRunning(true, false);
258       secondSplit = false;
259       firstSplitCompleted = false;
260       callRollBack = false;
261       cluster.getMaster().setCatalogJanitorEnabled(true);
262       TESTING_UTIL.deleteTable(tableName);
263     }
264   }
265 
266   @Test(timeout = 60000)
267   public void testRITStateForRollback() throws Exception {
268     final TableName tableName =
269         TableName.valueOf("testRITStateForRollback");
270     try {
271       // Create table then get the single region for our new table.
272       HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
273       final List<HRegion> regions = cluster.getRegions(tableName);
274       final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
275       insertData(tableName.getName(), admin, t);
276       t.close();
277 
278       // Turn off balancer so it doesn't cut in and mess up our placements.
279       this.admin.setBalancerRunning(false, true);
280       // Turn off the meta scanner so it don't remove parent on us.
281       cluster.getMaster().setCatalogJanitorEnabled(false);
282 
283       // find a splittable region
284       final HRegion region = findSplittableRegion(regions);
285       assertTrue("not able to find a splittable region", region != null);
286 
287       // install region co-processor to fail splits
288       region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
289         Coprocessor.PRIORITY_USER, region.getBaseConf());
290 
291       // split async
292       this.admin.split(region.getRegionName(), new byte[] {42});
293 
294       // we have to wait until the SPLITTING state is seen by the master
295       FailingSplitRegionObserver.latch.await();
296 
297       LOG.info("Waiting for region to come out of RIT");
298       TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
299         @Override
300         public boolean evaluate() throws Exception {
301           RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
302           Map<String, RegionState> rit = regionStates.getRegionsInTransition();
303           return !rit.containsKey(hri.getEncodedName());
304         }
305       });
306     } finally {
307       admin.setBalancerRunning(true, false);
308       cluster.getMaster().setCatalogJanitorEnabled(true);
309       TESTING_UTIL.deleteTable(tableName);
310     }
311   }
312 
313   public static class FailingSplitRegionObserver extends BaseRegionObserver {
314     static volatile CountDownLatch latch = new CountDownLatch(1);
315     @Override
316     public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
317         byte[] splitKey, List<Mutation> metaEntries) throws IOException {
318       latch.countDown();
319       throw new IOException("Causing rollback of region split");
320     }
321   }
322 
323  /**
324    * A test that intentionally has master fail the processing of the split message.
325    * Tests that the regionserver split ephemeral node gets cleaned up if it
326    * crashes and that after we process server shutdown, the daughters are up on
327    * line.
328    * @throws IOException
329    * @throws InterruptedException
330    * @throws NodeExistsException
331    * @throws KeeperException
332    * @throws DeserializationException
333    */
334   @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
335   throws IOException, InterruptedException, NodeExistsException, KeeperException,
336       DeserializationException, ServiceException {
337     final byte [] tableName =
338       Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
339 
340     // Create table then get the single region for our new table.
341     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
342     List<HRegion> regions = cluster.getRegions(tableName);
343     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
344 
345     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
346 
347     // Turn off balancer so it doesn't cut in and mess up our placements.
348     this.admin.setBalancerRunning(false, true);
349     // Turn off the meta scanner so it don't remove parent on us.
350     cluster.getMaster().setCatalogJanitorEnabled(false);
351     try {
352       // Add a bit of load up into the table so splittable.
353       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
354       // Get region pre-split.
355       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
356       printOutRegions(server, "Initial regions: ");
357       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
358       // Now, before we split, set special flag in master, a flag that has
359       // it FAIL the processing of split.
360       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
361       // Now try splitting and it should work.
362       split(hri, server, regionCount);
363         // Assert the ephemeral node is up in zk.
364       String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
365         hri.getEncodedName());
366       RegionTransition rt = null;
367       Stat stats = null;
368       List<HRegion> daughters = null;
369       if (useZKForAssignment) {
370         daughters = checkAndGetDaughters(tableName);
371 
372         // Wait till the znode moved to SPLIT
373         for (int i=0; i<100; i++) {
374           stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
375           rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
376             hri.getEncodedName()));
377           if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
378           Thread.sleep(100);
379         }
380         LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
381         assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
382         // Now crash the server
383         cluster.abortRegionServer(tableRegionIndex);
384       }
385       waitUntilRegionServerDead();
386       awaitDaughters(tableName, 2);
387       if (useZKForAssignment) {
388         regions = cluster.getRegions(tableName);
389         for (HRegion r: regions) {
390           assertTrue(daughters.contains(r));
391         }
392 
393         // Finally assert that the ephemeral SPLIT znode was cleaned up.
394         for (int i=0; i<100; i++) {
395           // wait a bit (10s max) for the node to disappear
396           stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
397           if (stats == null) break;
398           Thread.sleep(100);
399         }
400         LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
401         assertTrue(stats == null);
402       }
403     } finally {
404       // Set this flag back.
405       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
406       admin.setBalancerRunning(true, false);
407       cluster.getMaster().setCatalogJanitorEnabled(true);
408       cluster.startRegionServer();
409       t.close();
410     }
411   }
412 
413   @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
414   throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
415     final byte [] tableName =
416       Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
417 
418     // Create table then get the single region for our new table.
419     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
420     List<HRegion> regions = cluster.getRegions(tableName);
421     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
422 
423     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
424 
425     RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
426 
427     // Turn off balancer so it doesn't cut in and mess up our placements.
428     this.admin.setBalancerRunning(false, true);
429     // Turn off the meta scanner so it don't remove parent on us.
430     cluster.getMaster().setCatalogJanitorEnabled(false);
431     try {
432       // Add a bit of load up into the table so splittable.
433       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
434       // Get region pre-split.
435       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
436       printOutRegions(server, "Initial regions: ");
437       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
438       // Insert into zk a blocking znode, a znode of same name as region
439       // so it gets in way of our splitting.
440       ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
441       if (useZKForAssignment) {
442         ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
443           hri, fakedServer);
444       } else {
445         regionStates.updateRegionState(hri, RegionState.State.CLOSING);
446       }
447       // Now try splitting.... should fail.  And each should successfully
448       // rollback.
449       this.admin.split(hri.getRegionNameAsString());
450       this.admin.split(hri.getRegionNameAsString());
451       this.admin.split(hri.getRegionNameAsString());
452       // Wait around a while and assert count of regions remains constant.
453       for (int i = 0; i < 10; i++) {
454         Thread.sleep(100);
455         assertEquals(regionCount, ProtobufUtil.getOnlineRegions(server).size());
456       }
457       if (useZKForAssignment) {
458         // Now clear the zknode
459         ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
460           hri, fakedServer);
461       } else {
462         regionStates.regionOnline(hri, server.getServerName());
463       }
464       // Now try splitting and it should work.
465       split(hri, server, regionCount);
466       // Get daughters
467       checkAndGetDaughters(tableName);
468       // OK, so split happened after we cleared the blocking node.
469     } finally {
470       admin.setBalancerRunning(true, false);
471       cluster.getMaster().setCatalogJanitorEnabled(true);
472       t.close();
473     }
474   }
475 
476   /**
477    * Test that if daughter split on us, we won't do the shutdown handler fixup
478    * just because we can't find the immediate daughter of an offlined parent.
479    * @throws IOException
480    * @throws InterruptedException
481    */
482   @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
483   throws IOException, InterruptedException, ServiceException {
484     final byte [] tableName =
485       Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
486 
487     // Create table then get the single region for our new table.
488     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
489     List<HRegion> regions = cluster.getRegions(tableName);
490     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
491 
492     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
493 
494     // Turn off balancer so it doesn't cut in and mess up our placements.
495     this.admin.setBalancerRunning(false, true);
496     // Turn off the meta scanner so it don't remove parent on us.
497     cluster.getMaster().setCatalogJanitorEnabled(false);
498     try {
499       // Add a bit of load up into the table so splittable.
500       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
501       // Get region pre-split.
502       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
503       printOutRegions(server, "Initial regions: ");
504       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
505       // Now split.
506       split(hri, server, regionCount);
507       // Get daughters
508       List<HRegion> daughters = checkAndGetDaughters(tableName);
509       // Now split one of the daughters.
510       regionCount = ProtobufUtil.getOnlineRegions(server).size();
511       HRegionInfo daughter = daughters.get(0).getRegionInfo();
512       LOG.info("Daughter we are going to split: " + daughter);
513       // Compact first to ensure we have cleaned up references -- else the split
514       // will fail.
515       this.admin.compact(daughter.getRegionName());
516       daughters = cluster.getRegions(tableName);
517       HRegion daughterRegion = null;
518       for (HRegion r: daughters) {
519         if (r.getRegionInfo().equals(daughter)) {
520           daughterRegion = r;
521           LOG.info("Found matching HRI: " + daughterRegion);
522           break;
523         }
524       }
525       assertTrue(daughterRegion != null);
526       for (int i=0; i<100; i++) {
527         if (!daughterRegion.hasReferences()) break;
528         Threads.sleep(100);
529       }
530       assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
531       LOG.info("Daughter hri before split (has been compacted): " + daughter);
532       split(daughter, server, regionCount);
533       // Get list of daughters
534       daughters = cluster.getRegions(tableName);
535       for (HRegion d: daughters) {
536         LOG.info("Regions before crash: " + d);
537       }
538       // Now crash the server
539       cluster.abortRegionServer(tableRegionIndex);
540       waitUntilRegionServerDead();
541       awaitDaughters(tableName, daughters.size());
542       // Assert daughters are online and ONLY the original daughters -- that
543       // fixup didn't insert one during server shutdown recover.
544       regions = cluster.getRegions(tableName);
545       for (HRegion d: daughters) {
546         LOG.info("Regions after crash: " + d);
547       }
548       assertEquals(daughters.size(), regions.size());
549       for (HRegion r: regions) {
550         LOG.info("Regions post crash " + r);
551         assertTrue("Missing region post crash " + r, daughters.contains(r));
552       }
553     } finally {
554       admin.setBalancerRunning(true, false);
555       cluster.getMaster().setCatalogJanitorEnabled(true);
556       t.close();
557     }
558   }
559 
560   @Test(timeout = 180000)
561   public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
562     Configuration conf = TESTING_UTIL.getConfiguration();
563     TableName userTableName =
564         TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
565     HTableDescriptor htd = new HTableDescriptor(userTableName);
566     HColumnDescriptor hcd = new HColumnDescriptor("col");
567     htd.addFamily(hcd);
568     admin.createTable(htd);
569     HTable table = new HTable(conf, userTableName);
570     try {
571       for (int i = 0; i <= 5; i++) {
572         String row = "row" + i;
573         Put p = new Put(row.getBytes());
574         String val = "Val" + i;
575         p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
576         table.put(p);
577         admin.flush(userTableName.getName());
578         Delete d = new Delete(row.getBytes());
579         // Do a normal delete
580         table.delete(d);
581         admin.flush(userTableName.getName());
582       }
583       admin.majorCompact(userTableName.getName());
584       List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
585           .getMaster().getAssignmentManager().getRegionStates()
586           .getRegionsOfTable(userTableName);
587       HRegionInfo hRegionInfo = regionsOfTable.get(0);
588       Put p = new Put("row6".getBytes());
589       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
590       table.put(p);
591       p = new Put("row7".getBytes());
592       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
593       table.put(p);
594       p = new Put("row8".getBytes());
595       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
596       table.put(p);
597       admin.flush(userTableName.getName());
598       admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
599       regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
600           .getAssignmentManager().getRegionStates()
601           .getRegionsOfTable(userTableName);
602 
603       while (regionsOfTable.size() != 2) {
604         Thread.sleep(2000);
605         regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
606             .getAssignmentManager().getRegionStates()
607             .getRegionsOfTable(userTableName);
608       }
609       Assert.assertEquals(2, regionsOfTable.size());
610       Scan s = new Scan();
611       ResultScanner scanner = table.getScanner(s);
612       int mainTableCount = 0;
613       for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
614         mainTableCount++;
615       }
616       Assert.assertEquals(3, mainTableCount);
617     } finally {
618       table.close();
619     }
620   }
621 
622   /**
623    * Noop Abortable implementation used below in tests.
624    */
625   static class UselessTestAbortable implements Abortable {
626     boolean aborted = false;
627     @Override
628     public void abort(String why, Throwable e) {
629       LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
630       aborted = true;
631     }
632 
633     @Override
634     public boolean isAborted() {
635       return this.aborted;
636     }
637   }
638 
639   /**
640    * Verifies HBASE-5806.  When splitting is partially done and the master goes down
641    * when the SPLIT node is in either SPLIT or SPLITTING state.
642    *
643    * @throws IOException
644    * @throws InterruptedException
645    * @throws NodeExistsException
646    * @throws KeeperException
647    * @throws DeserializationException
648    */
649   @Test(timeout = 400000)
650   public void testMasterRestartWhenSplittingIsPartial()
651       throws IOException, InterruptedException, NodeExistsException,
652       KeeperException, DeserializationException, ServiceException {
653     final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
654 
655     if (!useZKForAssignment) {
656       // This test doesn't apply if not using ZK for assignment
657       return;
658     }
659 
660     // Create table then get the single region for our new table.
661     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
662     List<HRegion> regions = cluster.getRegions(tableName);
663     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
664 
665     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
666 
667     // Turn off balancer so it doesn't cut in and mess up our placements.
668     this.admin.setBalancerRunning(false, true);
669     // Turn off the meta scanner so it don't remove parent on us.
670     cluster.getMaster().setCatalogJanitorEnabled(false);
671     ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
672       "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
673     try {
674       // Add a bit of load up into the table so splittable.
675       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
676       // Get region pre-split.
677       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
678       printOutRegions(server, "Initial regions: ");
679       // Now, before we split, set special flag in master, a flag that has
680       // it FAIL the processing of split.
681       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
682       // Now try splitting and it should work.
683 
684       this.admin.split(hri.getRegionNameAsString());
685       checkAndGetDaughters(tableName);
686       // Assert the ephemeral node is up in zk.
687       String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
688       Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
689       LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
690           + stats);
691       byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
692       RegionTransition rtd = RegionTransition.parseFrom(bytes);
693       // State could be SPLIT or SPLITTING.
694       assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
695           || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
696 
697       // abort and wait for new master.
698       MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
699 
700       this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
701 
702       // Update the region to be offline and split, so that HRegionInfo#equals
703       // returns true in checking rebuilt region states map.
704       hri.setOffline(true);
705       hri.setSplit(true);
706       ServerName regionServerOfRegion = master.getAssignmentManager()
707         .getRegionStates().getRegionServerOfRegion(hri);
708       assertTrue(regionServerOfRegion != null);
709 
710       // Remove the block so that split can move ahead.
711       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
712       String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
713       Stat stat = new Stat();
714       byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
715       // ZKUtil.create
716       for (int i=0; data != null && i<60; i++) {
717         Thread.sleep(1000);
718         data = ZKUtil.getDataNoWatch(zkw, node, stat);
719       }
720       assertNull("Waited too long for ZK node to be removed: "+node, data);
721       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
722       assertTrue("Split parent should be in SPLIT state",
723         regionStates.isRegionInState(hri, State.SPLIT));
724       regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
725       assertTrue(regionServerOfRegion == null);
726     } finally {
727       // Set this flag back.
728       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
729       admin.setBalancerRunning(true, false);
730       cluster.getMaster().setCatalogJanitorEnabled(true);
731       t.close();
732       zkw.close();
733     }
734   }
735 
736   /**
737    * Verifies HBASE-5806.  Here the case is that splitting is completed but before the
738    * CJ could remove the parent region the master is killed and restarted.
739    * @throws IOException
740    * @throws InterruptedException
741    * @throws NodeExistsException
742    * @throws KeeperException
743    */
744   @Test (timeout = 300000)
745   public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
746       throws IOException, InterruptedException, NodeExistsException,
747       KeeperException, ServiceException {
748     final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
749 
750     // Create table then get the single region for our new table.
751     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
752     List<HRegion> regions = cluster.getRegions(tableName);
753     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
754 
755     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
756 
757     // Turn off balancer so it doesn't cut in and mess up our placements.
758     this.admin.setBalancerRunning(false, true);
759     // Turn off the meta scanner so it don't remove parent on us.
760     cluster.getMaster().setCatalogJanitorEnabled(false);
761     ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
762       "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
763     try {
764       // Add a bit of load up into the table so splittable.
765       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
766       // Get region pre-split.
767       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
768       printOutRegions(server, "Initial regions: ");
769 
770       this.admin.split(hri.getRegionNameAsString());
771       checkAndGetDaughters(tableName);
772       // Assert the ephemeral node is up in zk.
773       String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
774       Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
775       LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
776           + stats);
777       String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
778       Stat stat = new Stat();
779       byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
780       // ZKUtil.create
781       for (int i=0; data != null && i<60; i++) {
782         Thread.sleep(1000);
783         data = ZKUtil.getDataNoWatch(zkw, node, stat);
784       }
785       assertNull("Waited too long for ZK node to be removed: "+node, data);
786 
787       MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
788 
789       this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
790 
791       // Update the region to be offline and split, so that HRegionInfo#equals
792       // returns true in checking rebuilt region states map.
793       hri.setOffline(true);
794       hri.setSplit(true);
795       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
796       assertTrue("Split parent should be in SPLIT state",
797         regionStates.isRegionInState(hri, State.SPLIT));
798       ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
799       assertTrue(regionServerOfRegion == null);
800     } finally {
801       this.admin.setBalancerRunning(true, false);
802       cluster.getMaster().setCatalogJanitorEnabled(true);
803       t.close();
804       zkw.close();
805     }
806   }
807 
808   /**
809    *
810    * While transitioning node from RS_ZK_REGION_SPLITTING to
811    * RS_ZK_REGION_SPLITTING during region split,if zookeper went down split always
812    * fails for the region. HBASE-6088 fixes this scenario.
813    * This test case is to test the znode is deleted(if created) or not in roll back.
814    *
815    * @throws IOException
816    * @throws InterruptedException
817    * @throws KeeperException
818    */
819   @Test(timeout = 60000)
820   public void testSplitBeforeSettingSplittingInZK() throws Exception,
821       InterruptedException, KeeperException {
822     testSplitBeforeSettingSplittingInZKInternals();
823   }
824 
825   @Test(timeout = 60000)
826   public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
827     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
828     final TableName tableName =
829         TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
830     // Create table then get the single region for our new table.
831     HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
832     List<HRegion> regions = null;
833     try {
834       regions = cluster.getRegions(tableName);
835       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
836       HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
837       insertData(tableName.getName(), admin, t);
838       // Turn off balancer so it doesn't cut in and mess up our placements.
839       admin.setBalancerRunning(false, true);
840       // Turn off the meta scanner so it don't remove parent on us.
841       cluster.getMaster().setCatalogJanitorEnabled(false);
842       boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
843           tableName);
844       assertEquals("The specified table should present.", true, tableExists);
845       final HRegion region = findSplittableRegion(regions);
846       assertTrue("not able to find a splittable region", region != null);
847       SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
848       try {
849         st.prepare();
850         st.createDaughters(regionServer, regionServer);
851       } catch (IOException e) {
852 
853       }
854       tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
855           tableName);
856       assertEquals("The specified table should present.", true, tableExists);
857     } finally {
858       if (regions != null) {
859         String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
860             .getEncodedName());
861         ZKUtil.deleteNodeFailSilent(zkw, node);
862       }
863       admin.setBalancerRunning(true, false);
864       cluster.getMaster().setCatalogJanitorEnabled(true);
865       t.close();
866     }
867   }
868 
869   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
870       InterruptedException {
871     Put p = new Put(Bytes.toBytes("row1"));
872     p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
873     t.put(p);
874     p = new Put(Bytes.toBytes("row2"));
875     p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
876     t.put(p);
877     p = new Put(Bytes.toBytes("row3"));
878     p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
879     t.put(p);
880     p = new Put(Bytes.toBytes("row4"));
881     p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
882     t.put(p);
883     admin.flush(tableName);
884   }
885 
886   /**
887    * If a table has regions that have no store files in a region, they should split successfully
888    * into two regions with no store files.
889    */
890   @Test(timeout = 60000)
891   public void testSplitRegionWithNoStoreFiles()
892       throws Exception {
893     final TableName tableName =
894         TableName.valueOf("testSplitRegionWithNoStoreFiles");
895     // Create table then get the single region for our new table.
896     createTableAndWait(tableName.getName(), HConstants.CATALOG_FAMILY);
897     List<HRegion> regions = cluster.getRegions(tableName);
898     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
899     ensureTableRegionNotOnSameServerAsMeta(admin, hri);
900     int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
901     HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
902     // Turn off balancer so it doesn't cut in and mess up our placements.
903     this.admin.setBalancerRunning(false, true);
904     // Turn off the meta scanner so it don't remove parent on us.
905     cluster.getMaster().setCatalogJanitorEnabled(false);
906     try {
907       // Precondition: we created a table with no data, no store files.
908       printOutRegions(regionServer, "Initial regions: ");
909       Configuration conf = cluster.getConfiguration();
910       HBaseFsck.debugLsr(conf, new Path("/"));
911       Path rootDir = FSUtils.getRootDir(conf);
912       FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
913       Map<String, Path> storefiles =
914           FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
915       assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
916 
917       // find a splittable region.  Refresh the regions list
918       regions = cluster.getRegions(tableName);
919       final HRegion region = findSplittableRegion(regions);
920       assertTrue("not able to find a splittable region", region != null);
921 
922       // Now split.
923       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
924       try {
925         st.prepare();
926         st.execute(regionServer, regionServer);
927       } catch (IOException e) {
928         fail("Split execution should have succeeded with no exceptions thrown");
929       }
930 
931       // Postcondition: split the table with no store files into two regions, but still have not
932       // store files
933       List<HRegion> daughters = cluster.getRegions(tableName);
934       assertTrue(daughters.size() == 2);
935 
936       // check dirs
937       HBaseFsck.debugLsr(conf, new Path("/"));
938       Map<String, Path> storefilesAfter =
939           FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
940       assertEquals("Expected nothing but found " + storefilesAfter.toString(),
941           storefilesAfter.size(), 0);
942 
943       hri = region.getRegionInfo(); // split parent
944       AssignmentManager am = cluster.getMaster().getAssignmentManager();
945       RegionStates regionStates = am.getRegionStates();
946       long start = EnvironmentEdgeManager.currentTimeMillis();
947       while (!regionStates.isRegionInState(hri, State.SPLIT)) {
948         assertFalse("Timed out in waiting split parent to be in state SPLIT",
949           EnvironmentEdgeManager.currentTimeMillis() - start > 60000);
950         Thread.sleep(500);
951       }
952 
953       // We should not be able to assign it again
954       am.assign(hri, true, true);
955       assertFalse("Split region can't be assigned",
956         regionStates.isRegionInTransition(hri));
957       assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
958 
959       // We should not be able to unassign it either
960       am.unassign(hri, true, null);
961       assertFalse("Split region can't be unassigned",
962         regionStates.isRegionInTransition(hri));
963       assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
964     } finally {
965       admin.setBalancerRunning(true, false);
966       cluster.getMaster().setCatalogJanitorEnabled(true);
967     }
968   }
969 
970   @Test(timeout = 180000)
971   public void testSplitHooksBeforeAndAfterPONR() throws Exception {
972     TableName firstTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1");
973     TableName secondTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2");
974     HColumnDescriptor hcd = new HColumnDescriptor("cf");
975 
976     HTableDescriptor desc = new HTableDescriptor(firstTable);
977     desc.addCoprocessor(MockedRegionObserver.class.getName());
978     desc.addFamily(hcd);
979     admin.createTable(desc);
980     TESTING_UTIL.waitUntilAllRegionsAssigned(firstTable);
981 
982     desc = new HTableDescriptor(secondTable);
983     desc.addFamily(hcd);
984     admin.createTable(desc);
985     TESTING_UTIL.waitUntilAllRegionsAssigned(secondTable);
986 
987     List<HRegion> firstTableRegions = cluster.getRegions(firstTable);
988     List<HRegion> secondTableRegions = cluster.getRegions(secondTable);
989 
990     // Check that both tables actually have regions.
991     if (firstTableRegions.size() == 0 || secondTableRegions.size() == 0) {
992       fail("Each table should have at least one region.");
993     }
994     ServerName serverName =
995         cluster.getServerHoldingRegion(firstTableRegions.get(0).getRegionName());
996     admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
997       Bytes.toBytes(serverName.getServerName()));
998     HTable table1 = null;
999     HTable table2 = null;
1000     try {
1001       table1 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1002       table2 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1003       insertData(firstTable.getName(), admin, table1);
1004       insertData(secondTable.getName(), admin, table2);
1005       admin.split(firstTable.getName(), "row2".getBytes());
1006       firstTableRegions = cluster.getRegions(firstTable.getName());
1007       while (firstTableRegions.size() != 2) {
1008         Thread.sleep(1000);
1009         firstTableRegions = cluster.getRegions(firstTable.getName());
1010       }
1011       assertEquals("Number of regions after split should be 2.", 2, firstTableRegions.size());
1012       secondTableRegions = cluster.getRegions(secondTable.getName());
1013       assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
1014     } finally {
1015       if (table1 != null) {
1016         table1.close();
1017       }
1018       if (table2 != null) {
1019         table2.close();
1020       }
1021       TESTING_UTIL.deleteTable(firstTable);
1022       TESTING_UTIL.deleteTable(secondTable);
1023     }
1024   }
1025 
1026   private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
1027     final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
1028     try {
1029       // Create table then get the single region for our new table.
1030       createTableAndWait(tableName, Bytes.toBytes("cf"));
1031 
1032       List<HRegion> regions = awaitTableRegions(tableName);
1033       assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
1034 
1035       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
1036       HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1037       final HRegion region = findSplittableRegion(regions);
1038       assertTrue("not able to find a splittable region", region != null);
1039       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
1040         @Override
1041         public PairOfSameType<HRegion> stepsBeforePONR(final Server server,
1042             final RegionServerServices services, boolean testing) throws IOException {
1043           throw new SplittingNodeCreationFailedException ();
1044         }
1045       };
1046       String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1047           region.getRegionInfo().getEncodedName());
1048       regionServer.getZooKeeper().sync(node);
1049       for (int i = 0; i < 100; i++) {
1050         // We expect the znode to be deleted by this time. Here the
1051         // znode could be in OPENED state and the
1052         // master has not yet deleted the znode.
1053         if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
1054           Thread.sleep(100);
1055         }
1056       }
1057       try {
1058         st.prepare();
1059         st.execute(regionServer, regionServer);
1060       } catch (IOException e) {
1061         // check for the specific instance in case the Split failed due to the
1062         // existence of the znode in OPENED state.
1063         // This will at least make the test to fail;
1064         assertTrue("Should be instance of CreateSplittingNodeFailedException",
1065             e instanceof SplittingNodeCreationFailedException );
1066         node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1067             region.getRegionInfo().getEncodedName());
1068         {
1069           assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1070         }
1071         assertTrue(st.rollback(regionServer, regionServer));
1072         assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1073       }
1074     } finally {
1075       TESTING_UTIL.deleteTable(tableName);
1076     }
1077   }
1078 
1079   public static class MockedSplitTransaction extends SplitTransaction {
1080 
1081     private HRegion currentRegion;
1082     public MockedSplitTransaction(HRegion r, byte[] splitrow) {
1083       super(r, splitrow);
1084       this.currentRegion = r;
1085     }
1086 
1087     @Override
1088     void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b)
1089         throws IOException {
1090       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1091           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1092         try {
1093           if (!secondSplit){
1094             callRollBack = true;
1095             latch.await();
1096           }
1097         } catch (InterruptedException e) {
1098         }
1099 
1100       }
1101       super.transitionZKNode(server, services, a, b);
1102       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1103           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1104         firstSplitCompleted = true;
1105       }
1106     }
1107     @Override
1108     public boolean rollback(Server server, RegionServerServices services) throws IOException {
1109       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1110           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1111         if(secondSplit){
1112           super.rollback(server, services);
1113           latch.countDown();
1114           return true;
1115         }
1116       }
1117       return super.rollback(server, services);
1118     }
1119 
1120   }
1121 
1122   private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
1123     for (int i = 0; i < 5; ++i) {
1124       for (HRegion r: regions) {
1125         if (r.isSplittable()) {
1126           return(r);
1127         }
1128       }
1129       Thread.sleep(100);
1130     }
1131     return(null);
1132   }
1133 
1134   private List<HRegion> checkAndGetDaughters(byte[] tableName)
1135       throws InterruptedException {
1136     List<HRegion> daughters = null;
1137     // try up to 10s
1138     for (int i=0; i<100; i++) {
1139       daughters = cluster.getRegions(tableName);
1140       if (daughters.size() >= 2) break;
1141       Thread.sleep(100);
1142     }
1143     assertTrue(daughters.size() >= 2);
1144     return daughters;
1145   }
1146 
1147   private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
1148   throws IOException, InterruptedException {
1149     cluster.abortMaster(0);
1150     cluster.waitOnMaster(0);
1151     cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
1152     		MockMasterWithoutCatalogJanitor.class, HMaster.class);
1153     MockMasterWithoutCatalogJanitor master = null;
1154     master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
1155     cluster.waitForActiveAndReadyMaster();
1156     return master;
1157   }
1158 
1159   private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
1160       throws IOException, InterruptedException {
1161     this.admin.split(hri.getRegionNameAsString());
1162     try {
1163       for (int i = 0; ProtobufUtil.getOnlineRegions(server).size() <= regionCount && i < 300; i++) {
1164         LOG.debug("Waiting on region to split");
1165         Thread.sleep(100);
1166       }
1167 
1168       assertFalse("Waited too long for split",
1169         ProtobufUtil.getOnlineRegions(server).size() <= regionCount);
1170     } catch (RegionServerStoppedException e) {
1171       if (useZKForAssignment) {
1172         // If not using ZK for assignment, the exception may be expected.
1173         LOG.error(e);
1174         throw e;
1175       }
1176     }
1177   }
1178 
1179   /**
1180    * Ensure single table region is not on same server as the single hbase:meta table
1181    * region.
1182    * @param admin
1183    * @param hri
1184    * @return Index of the server hosting the single table region
1185    * @throws UnknownRegionException
1186    * @throws MasterNotRunningException
1187    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
1188    * @throws InterruptedException
1189    */
1190   private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
1191       final HRegionInfo hri)
1192   throws HBaseIOException, MasterNotRunningException,
1193   ZooKeeperConnectionException, InterruptedException {
1194     // Now make sure that the table region is not on same server as that hosting
1195     // hbase:meta  We don't want hbase:meta replay polluting our test when we later crash
1196     // the table region serving server.
1197     int metaServerIndex = cluster.getServerWithMeta();
1198     assertTrue(metaServerIndex != -1);
1199     HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1200     int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1201     assertTrue(tableRegionIndex != -1);
1202     HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1203     if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1204       HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1205       assertNotNull(hrs);
1206       assertNotNull(hri);
1207       LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1208         metaRegionServer.getServerName() + " to " +
1209         hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1210       admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1211     }
1212     // Wait till table region is up on the server that is NOT carrying hbase:meta.
1213     for (int i = 0; i < 100; i++) {
1214       tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1215       if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1216       LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1217         tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1218       Thread.sleep(100);
1219     }
1220     assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1221         && tableRegionIndex != metaServerIndex);
1222     // Verify for sure table region is not on same server as hbase:meta
1223     tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1224     assertTrue(tableRegionIndex != -1);
1225     assertNotSame(metaServerIndex, tableRegionIndex);
1226     return tableRegionIndex;
1227   }
1228 
1229   /**
1230    * Find regionserver other than the one passed.
1231    * Can't rely on indexes into list of regionservers since crashed servers
1232    * occupy an index.
1233    * @param cluster
1234    * @param notThisOne
1235    * @return A regionserver that is not <code>notThisOne</code> or null if none
1236    * found
1237    */
1238   private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1239       final HRegionServer notThisOne) {
1240     for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1241       HRegionServer hrs = rst.getRegionServer();
1242       if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1243       if (hrs.isStopping() || hrs.isStopped()) continue;
1244       return hrs;
1245     }
1246     return null;
1247   }
1248 
1249   private void printOutRegions(final HRegionServer hrs, final String prefix)
1250       throws IOException {
1251     List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
1252     for (HRegionInfo region: regions) {
1253       LOG.info(prefix + region.getRegionNameAsString());
1254     }
1255   }
1256 
1257   private void waitUntilRegionServerDead() throws InterruptedException {
1258     // Wait until the master processes the RS shutdown
1259     for (int i=0; cluster.getMaster().getClusterStatus().
1260         getServers().size() == NB_SERVERS && i<100; i++) {
1261       LOG.info("Waiting on server to go down");
1262       Thread.sleep(100);
1263     }
1264     assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1265         getServers().size() == NB_SERVERS);
1266   }
1267 
1268   private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
1269     // Wait till regions are back on line again.
1270     for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1271       LOG.info("Waiting for repair to happen");
1272       Thread.sleep(1000);
1273     }
1274     if (cluster.getRegions(tableName).size() < numDaughters) {
1275       fail("Waiting too long for daughter regions");
1276     }
1277   }
1278 
1279   private List<HRegion> awaitTableRegions(final byte[] tableName) throws InterruptedException {
1280     List<HRegion> regions = null;
1281     for (int i = 0; i < 100; i++) {
1282       regions = cluster.getRegions(tableName);
1283       if (regions.size() > 0) break;
1284       Thread.sleep(100);
1285     }
1286     return regions;
1287   }
1288 
1289   private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException,
1290       InterruptedException {
1291     HTable t = TESTING_UTIL.createTable(tableName, cf);
1292     awaitTableRegions(tableName);
1293     assertTrue("Table not online: " + Bytes.toString(tableName),
1294       cluster.getRegions(tableName).size() != 0);
1295     return t;
1296   }
1297 
1298   public static class MockMasterWithoutCatalogJanitor extends HMaster {
1299 
1300     public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException,
1301         InterruptedException {
1302       super(conf);
1303     }
1304   }
1305 
1306   private static class SplittingNodeCreationFailedException  extends IOException {
1307     private static final long serialVersionUID = 1652404976265623004L;
1308 
1309     public SplittingNodeCreationFailedException () {
1310       super();
1311     }
1312   }
1313 
1314   public static class MockedRegionObserver extends BaseRegionObserver {
1315     private SplitTransaction st = null;
1316     private PairOfSameType<HRegion> daughterRegions = null;
1317 
1318     @Override
1319     public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
1320         byte[] splitKey, List<Mutation> metaEntries) throws IOException {
1321       RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1322       HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1323       List<HRegion> onlineRegions =
1324           rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
1325       HRegion region = onlineRegions.get(0);
1326       for (HRegion r : onlineRegions) {
1327         if (r.getRegionInfo().containsRow(splitKey)) {
1328           region = r;
1329           break;
1330         }
1331       }
1332       st = new SplitTransaction(region, splitKey);
1333       if (!st.prepare()) {
1334         LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
1335             + " failed. So returning null. ");
1336         ctx.bypass();
1337         return;
1338       }
1339       region.forceSplit(splitKey);
1340       daughterRegions = st.stepsBeforePONR(rs, rs, false);
1341       HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
1342       copyOfParent.setOffline(true);
1343       copyOfParent.setSplit(true);
1344       // Put for parent
1345       Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
1346       MetaEditor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
1347         daughterRegions.getSecond().getRegionInfo());
1348       metaEntries.add(putParent);
1349       // Puts for daughters
1350       Put putA = MetaEditor.makePutFromRegionInfo(daughterRegions.getFirst().getRegionInfo());
1351       Put putB = MetaEditor.makePutFromRegionInfo(daughterRegions.getSecond().getRegionInfo());
1352       st.addLocation(putA, rs.getServerName(), 1);
1353       st.addLocation(putB, rs.getServerName(), 1);
1354       metaEntries.add(putA);
1355       metaEntries.add(putB);
1356     }
1357 
1358     @Override
1359     public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
1360         throws IOException {
1361       RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1362       HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1363       st.stepsAfterPONR(rs, rs, daughterRegions);
1364     }
1365 
1366   }
1367 }
1368