View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNotSame;
25  import static org.junit.Assert.assertNull;
26  import static org.junit.Assert.assertTrue;
27  import static org.junit.Assert.fail;
28  
29  import java.io.IOException;
30  import java.util.Collection;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.concurrent.CountDownLatch;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.Abortable;
41  import org.apache.hadoop.hbase.Coprocessor;
42  import org.apache.hadoop.hbase.HBaseIOException;
43  import org.apache.hadoop.hbase.HBaseTestingUtility;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HTableDescriptor;
48  import org.apache.hadoop.hbase.testclassification.LargeTests;
49  import org.apache.hadoop.hbase.MasterNotRunningException;
50  import org.apache.hadoop.hbase.MiniHBaseCluster;
51  import org.apache.hadoop.hbase.RegionTransition;
52  import org.apache.hadoop.hbase.Server;
53  import org.apache.hadoop.hbase.ServerName;
54  import org.apache.hadoop.hbase.TableName;
55  import org.apache.hadoop.hbase.UnknownRegionException;
56  import org.apache.hadoop.hbase.Waiter;
57  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
58  import org.apache.hadoop.hbase.catalog.MetaEditor;
59  import org.apache.hadoop.hbase.catalog.MetaReader;
60  import org.apache.hadoop.hbase.client.Delete;
61  import org.apache.hadoop.hbase.client.HBaseAdmin;
62  import org.apache.hadoop.hbase.client.HTable;
63  import org.apache.hadoop.hbase.client.Mutation;
64  import org.apache.hadoop.hbase.client.Put;
65  import org.apache.hadoop.hbase.client.Result;
66  import org.apache.hadoop.hbase.client.ResultScanner;
67  import org.apache.hadoop.hbase.client.Scan;
68  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
69  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
70  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
71  import org.apache.hadoop.hbase.exceptions.DeserializationException;
72  import org.apache.hadoop.hbase.executor.EventType;
73  import org.apache.hadoop.hbase.master.AssignmentManager;
74  import org.apache.hadoop.hbase.master.HMaster;
75  import org.apache.hadoop.hbase.master.RegionState;
76  import org.apache.hadoop.hbase.master.RegionStates;
77  import org.apache.hadoop.hbase.master.RegionState.State;
78  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
79  import org.apache.hadoop.hbase.util.Bytes;
80  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
81  import org.apache.hadoop.hbase.util.FSUtils;
82  import org.apache.hadoop.hbase.util.HBaseFsck;
83  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
84  import org.apache.hadoop.hbase.util.PairOfSameType;
85  import org.apache.hadoop.hbase.util.Threads;
86  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
87  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
88  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
89  import org.apache.zookeeper.KeeperException;
90  import org.apache.zookeeper.KeeperException.NodeExistsException;
91  import org.apache.zookeeper.data.Stat;
92  import org.junit.After;
93  import org.junit.AfterClass;
94  import org.junit.Assert;
95  import org.junit.Before;
96  import org.junit.BeforeClass;
97  import org.junit.Test;
98  import org.junit.experimental.categories.Category;
99  
100 import com.google.protobuf.ServiceException;
101 
102 /**
103  * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction}
104  * only the below tests are against a running cluster where {@link TestSplitTransaction}
105  * is tests against a bare {@link HRegion}.
106  */
107 @Category(LargeTests.class)
108 public class TestSplitTransactionOnCluster {
109   private static final Log LOG =
110     LogFactory.getLog(TestSplitTransactionOnCluster.class);
111   private HBaseAdmin admin = null;
112   private MiniHBaseCluster cluster = null;
113   private static final int NB_SERVERS = 3;
114   private static CountDownLatch latch = new CountDownLatch(1);
115   private static volatile boolean secondSplit = false;
116   private static volatile boolean callRollBack = false;
117   private static volatile boolean firstSplitCompleted = false;
118   private static boolean useZKForAssignment = true;
119 
120   static final HBaseTestingUtility TESTING_UTIL =
121     new HBaseTestingUtility();
122 
123   static void setupOnce() throws Exception {
124     TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
125     useZKForAssignment =
126         TESTING_UTIL.getConfiguration().getBoolean("hbase.assignment.usezk", false);
127     TESTING_UTIL.startMiniCluster(NB_SERVERS);
128   }
129 
130   @BeforeClass public static void before() throws Exception {
131     // Use ZK for region assignment
132     TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
133     setupOnce();
134   }
135 
136   @AfterClass public static void after() throws Exception {
137     TESTING_UTIL.shutdownMiniCluster();
138   }
139 
140   @Before public void setup() throws IOException {
141     TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
142     this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
143     this.cluster = TESTING_UTIL.getMiniHBaseCluster();
144   }
145 
146   @After
147   public void tearDown() throws Exception {
148     this.admin.close();
149   }
150 
151   private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions) {
152     assertEquals(1, regions.size());
153     HRegionInfo hri = regions.get(0).getRegionInfo();
154     return waitOnRIT(hri);
155   }
156 
157   /**
158    * Often region has not yet fully opened.  If we try to use it -- do a move for instance -- it
159    * will fail silently if the region is not yet opened.
160    * @param hri Region to check if in Regions In Transition... wait until out of transition before
161    * returning
162    * @return Passed in <code>hri</code>
163    */
164   private HRegionInfo waitOnRIT(final HRegionInfo hri) {
165     // Close worked but we are going to open the region elsewhere.  Before going on, make sure
166     // this completes.
167     while (TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
168         getRegionStates().isRegionInTransition(hri)) {
169       LOG.info("Waiting on region in transition: " +
170         TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().
171           getRegionTransitionState(hri));
172       Threads.sleep(10);
173     }
174     return hri;
175   }
176 
177   @SuppressWarnings("deprecation")
178   @Test(timeout = 60000)
179   public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
180     final TableName tableName =
181         TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
182 
183     if (!useZKForAssignment) {
184       // This test doesn't apply if not using ZK for assignment
185       return;
186     }
187 
188     try {
189       // Create table then get the single region for our new table.
190       HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
191       final List<HRegion> regions = cluster.getRegions(tableName);
192       HRegionInfo hri = getAndCheckSingleTableRegion(regions);
193       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
194       final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
195       insertData(tableName.getName(), admin, t);
196       t.close();
197 
198       // Turn off balancer so it doesn't cut in and mess up our placements.
199       this.admin.setBalancerRunning(false, true);
200       // Turn off the meta scanner so it don't remove parent on us.
201       cluster.getMaster().setCatalogJanitorEnabled(false);
202 
203       // find a splittable region
204       final HRegion region = findSplittableRegion(regions);
205       assertTrue("not able to find a splittable region", region != null);
206 
207       new Thread() {
208         @Override
209         public void run() {
210           SplitTransaction st = null;
211           st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
212           try {
213             st.prepare();
214             st.execute(regionServer, regionServer);
215           } catch (IOException e) {
216 
217           }
218         }
219       }.start();
220       for (int i = 0; !callRollBack && i < 100; i++) {
221         Thread.sleep(100);
222       }
223       assertTrue("Waited too long for rollback", callRollBack);
224       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
225       try {
226         secondSplit = true;
227         // make region splittable
228         region.initialize();
229         st.prepare();
230         st.execute(regionServer, regionServer);
231       } catch (IOException e) {
232         LOG.debug("Rollback started :"+ e.getMessage());
233         st.rollback(regionServer, regionServer);
234       }
235       for (int i=0; !firstSplitCompleted && i<100; i++) {
236         Thread.sleep(100);
237       }
238       assertTrue("fist split did not complete", firstSplitCompleted);
239 
240       RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
241       Map<String, RegionState> rit = regionStates.getRegionsInTransition();
242 
243       for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
244         Thread.sleep(100);
245       }
246       assertFalse("region still in transition", rit.containsKey(
247           rit.containsKey(hri.getTable())));
248 
249       List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
250       // Region server side split is successful.
251       assertEquals("The parent region should be splitted", 2, onlineRegions.size());
252       //Should be present in RIT
253       List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
254           .getRegionStates().getRegionsOfTable(tableName);
255       // Master side should also reflect the same
256       assertEquals("No of regions in master", 2, regionsOfTable.size());
257     } finally {
258       admin.setBalancerRunning(true, false);
259       secondSplit = false;
260       firstSplitCompleted = false;
261       callRollBack = false;
262       cluster.getMaster().setCatalogJanitorEnabled(true);
263       TESTING_UTIL.deleteTable(tableName);
264     }
265   }
266 
267   @Test(timeout = 60000)
268   public void testRITStateForRollback() throws Exception {
269     final TableName tableName =
270         TableName.valueOf("testRITStateForRollback");
271     try {
272       // Create table then get the single region for our new table.
273       HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
274       final List<HRegion> regions = cluster.getRegions(tableName);
275       final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
276       insertData(tableName.getName(), admin, t);
277       t.close();
278 
279       // Turn off balancer so it doesn't cut in and mess up our placements.
280       this.admin.setBalancerRunning(false, true);
281       // Turn off the meta scanner so it don't remove parent on us.
282       cluster.getMaster().setCatalogJanitorEnabled(false);
283 
284       // find a splittable region
285       final HRegion region = findSplittableRegion(regions);
286       assertTrue("not able to find a splittable region", region != null);
287 
288       // install region co-processor to fail splits
289       region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
290         Coprocessor.PRIORITY_USER, region.getBaseConf());
291 
292       // split async
293       this.admin.split(region.getRegionName(), new byte[] {42});
294 
295       // we have to wait until the SPLITTING state is seen by the master
296       FailingSplitRegionObserver.latch.await();
297 
298       LOG.info("Waiting for region to come out of RIT");
299       TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
300         @Override
301         public boolean evaluate() throws Exception {
302           RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
303           Map<String, RegionState> rit = regionStates.getRegionsInTransition();
304           return !rit.containsKey(hri.getEncodedName());
305         }
306       });
307     } finally {
308       admin.setBalancerRunning(true, false);
309       cluster.getMaster().setCatalogJanitorEnabled(true);
310       TESTING_UTIL.deleteTable(tableName);
311     }
312   }
313 
314   public static class FailingSplitRegionObserver extends BaseRegionObserver {
315     static volatile CountDownLatch latch = new CountDownLatch(1);
316     @Override
317     public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
318         byte[] splitKey, List<Mutation> metaEntries) throws IOException {
319       latch.countDown();
320       throw new IOException("Causing rollback of region split");
321     }
322   }
323 
324  /**
325    * A test that intentionally has master fail the processing of the split message.
326    * Tests that the regionserver split ephemeral node gets cleaned up if it
327    * crashes and that after we process server shutdown, the daughters are up on
328    * line.
329    * @throws IOException
330    * @throws InterruptedException
331    * @throws NodeExistsException
332    * @throws KeeperException
333    * @throws DeserializationException
334    */
335   @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
336   throws IOException, InterruptedException, NodeExistsException, KeeperException,
337       DeserializationException, ServiceException {
338     final byte [] tableName =
339       Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
340 
341     // Create table then get the single region for our new table.
342     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
343     List<HRegion> regions = cluster.getRegions(tableName);
344     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
345 
346     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
347 
348     // Turn off balancer so it doesn't cut in and mess up our placements.
349     this.admin.setBalancerRunning(false, true);
350     // Turn off the meta scanner so it don't remove parent on us.
351     cluster.getMaster().setCatalogJanitorEnabled(false);
352     try {
353       // Add a bit of load up into the table so splittable.
354       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
355       // Get region pre-split.
356       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
357       printOutRegions(server, "Initial regions: ");
358       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
359       // Now, before we split, set special flag in master, a flag that has
360       // it FAIL the processing of split.
361       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
362       // Now try splitting and it should work.
363       split(hri, server, regionCount);
364         // Assert the ephemeral node is up in zk.
365       String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
366         hri.getEncodedName());
367       RegionTransition rt = null;
368       Stat stats = null;
369       List<HRegion> daughters = null;
370       if (useZKForAssignment) {
371         daughters = checkAndGetDaughters(tableName);
372 
373         // Wait till the znode moved to SPLIT
374         for (int i=0; i<100; i++) {
375           stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
376           rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
377             hri.getEncodedName()));
378           if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
379           Thread.sleep(100);
380         }
381         LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
382         assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
383         // Now crash the server
384         cluster.abortRegionServer(tableRegionIndex);
385       }
386       waitUntilRegionServerDead();
387       awaitDaughters(tableName, 2);
388       if (useZKForAssignment) {
389         regions = cluster.getRegions(tableName);
390         for (HRegion r: regions) {
391           assertTrue(daughters.contains(r));
392         }
393 
394         // Finally assert that the ephemeral SPLIT znode was cleaned up.
395         for (int i=0; i<100; i++) {
396           // wait a bit (10s max) for the node to disappear
397           stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
398           if (stats == null) break;
399           Thread.sleep(100);
400         }
401         LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
402         assertTrue(stats == null);
403       }
404     } finally {
405       // Set this flag back.
406       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
407       admin.setBalancerRunning(true, false);
408       cluster.getMaster().setCatalogJanitorEnabled(true);
409       cluster.startRegionServer();
410       t.close();
411     }
412   }
413 
414   @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
415   throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
416     final byte [] tableName =
417       Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
418 
419     // Create table then get the single region for our new table.
420     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
421     List<HRegion> regions = cluster.getRegions(tableName);
422     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
423 
424     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
425 
426     RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
427 
428     // Turn off balancer so it doesn't cut in and mess up our placements.
429     this.admin.setBalancerRunning(false, true);
430     // Turn off the meta scanner so it don't remove parent on us.
431     cluster.getMaster().setCatalogJanitorEnabled(false);
432     try {
433       // Add a bit of load up into the table so splittable.
434       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
435       // Get region pre-split.
436       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
437       printOutRegions(server, "Initial regions: ");
438       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
439       // Insert into zk a blocking znode, a znode of same name as region
440       // so it gets in way of our splitting.
441       ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
442       if (useZKForAssignment) {
443         ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
444           hri, fakedServer);
445       } else {
446         regionStates.updateRegionState(hri, RegionState.State.CLOSING);
447       }
448       // Now try splitting.... should fail.  And each should successfully
449       // rollback.
450       this.admin.split(hri.getRegionNameAsString());
451       this.admin.split(hri.getRegionNameAsString());
452       this.admin.split(hri.getRegionNameAsString());
453       // Wait around a while and assert count of regions remains constant.
454       for (int i = 0; i < 10; i++) {
455         Thread.sleep(100);
456         assertEquals(regionCount, ProtobufUtil.getOnlineRegions(server).size());
457       }
458       if (useZKForAssignment) {
459         // Now clear the zknode
460         ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
461           hri, fakedServer);
462       } else {
463         regionStates.regionOnline(hri, server.getServerName());
464       }
465       // Now try splitting and it should work.
466       split(hri, server, regionCount);
467       // Get daughters
468       checkAndGetDaughters(tableName);
469       // OK, so split happened after we cleared the blocking node.
470     } finally {
471       admin.setBalancerRunning(true, false);
472       cluster.getMaster().setCatalogJanitorEnabled(true);
473       t.close();
474     }
475   }
476 
477   /**
478    * Test that if daughter split on us, we won't do the shutdown handler fixup
479    * just because we can't find the immediate daughter of an offlined parent.
480    * @throws IOException
481    * @throws InterruptedException
482    */
483   @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
484   throws IOException, InterruptedException, ServiceException {
485     final byte [] tableName =
486       Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
487 
488     // Create table then get the single region for our new table.
489     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
490     List<HRegion> regions = cluster.getRegions(tableName);
491     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
492 
493     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
494 
495     // Turn off balancer so it doesn't cut in and mess up our placements.
496     this.admin.setBalancerRunning(false, true);
497     // Turn off the meta scanner so it don't remove parent on us.
498     cluster.getMaster().setCatalogJanitorEnabled(false);
499     try {
500       // Add a bit of load up into the table so splittable.
501       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
502       // Get region pre-split.
503       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
504       printOutRegions(server, "Initial regions: ");
505       int regionCount = ProtobufUtil.getOnlineRegions(server).size();
506       // Now split.
507       split(hri, server, regionCount);
508       // Get daughters
509       List<HRegion> daughters = checkAndGetDaughters(tableName);
510       // Now split one of the daughters.
511       regionCount = ProtobufUtil.getOnlineRegions(server).size();
512       HRegionInfo daughter = daughters.get(0).getRegionInfo();
513       LOG.info("Daughter we are going to split: " + daughter);
514       // Compact first to ensure we have cleaned up references -- else the split
515       // will fail.
516       this.admin.compact(daughter.getRegionName());
517       daughters = cluster.getRegions(tableName);
518       HRegion daughterRegion = null;
519       for (HRegion r: daughters) {
520         if (r.getRegionInfo().equals(daughter)) {
521           daughterRegion = r;
522           LOG.info("Found matching HRI: " + daughterRegion);
523           break;
524         }
525       }
526       assertTrue(daughterRegion != null);
527       for (int i=0; i<100; i++) {
528         if (!daughterRegion.hasReferences()) break;
529         Threads.sleep(100);
530       }
531       assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
532       LOG.info("Daughter hri before split (has been compacted): " + daughter);
533       split(daughter, server, regionCount);
534       // Get list of daughters
535       daughters = cluster.getRegions(tableName);
536       for (HRegion d: daughters) {
537         LOG.info("Regions before crash: " + d);
538       }
539       // Now crash the server
540       cluster.abortRegionServer(tableRegionIndex);
541       waitUntilRegionServerDead();
542       awaitDaughters(tableName, daughters.size());
543       // Assert daughters are online and ONLY the original daughters -- that
544       // fixup didn't insert one during server shutdown recover.
545       regions = cluster.getRegions(tableName);
546       for (HRegion d: daughters) {
547         LOG.info("Regions after crash: " + d);
548       }
549       assertEquals(daughters.size(), regions.size());
550       for (HRegion r: regions) {
551         LOG.info("Regions post crash " + r);
552         assertTrue("Missing region post crash " + r, daughters.contains(r));
553       }
554     } finally {
555       admin.setBalancerRunning(true, false);
556       cluster.getMaster().setCatalogJanitorEnabled(true);
557       t.close();
558     }
559   }
560 
561   @Test(timeout = 180000)
562   public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
563     Configuration conf = TESTING_UTIL.getConfiguration();
564     TableName userTableName =
565         TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
566     HTableDescriptor htd = new HTableDescriptor(userTableName);
567     HColumnDescriptor hcd = new HColumnDescriptor("col");
568     htd.addFamily(hcd);
569     admin.createTable(htd);
570     HTable table = new HTable(conf, userTableName);
571     try {
572       for (int i = 0; i <= 5; i++) {
573         String row = "row" + i;
574         Put p = new Put(row.getBytes());
575         String val = "Val" + i;
576         p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
577         table.put(p);
578         admin.flush(userTableName.getName());
579         Delete d = new Delete(row.getBytes());
580         // Do a normal delete
581         table.delete(d);
582         admin.flush(userTableName.getName());
583       }
584       admin.majorCompact(userTableName.getName());
585       List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
586           .getMaster().getAssignmentManager().getRegionStates()
587           .getRegionsOfTable(userTableName);
588       HRegionInfo hRegionInfo = regionsOfTable.get(0);
589       Put p = new Put("row6".getBytes());
590       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
591       table.put(p);
592       p = new Put("row7".getBytes());
593       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
594       table.put(p);
595       p = new Put("row8".getBytes());
596       p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
597       table.put(p);
598       admin.flush(userTableName.getName());
599       admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
600       regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
601           .getAssignmentManager().getRegionStates()
602           .getRegionsOfTable(userTableName);
603 
604       while (regionsOfTable.size() != 2) {
605         Thread.sleep(2000);
606         regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
607             .getAssignmentManager().getRegionStates()
608             .getRegionsOfTable(userTableName);
609       }
610       Assert.assertEquals(2, regionsOfTable.size());
611       Scan s = new Scan();
612       ResultScanner scanner = table.getScanner(s);
613       int mainTableCount = 0;
614       for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
615         mainTableCount++;
616       }
617       Assert.assertEquals(3, mainTableCount);
618     } finally {
619       table.close();
620     }
621   }
622 
623   /**
624    * Noop Abortable implementation used below in tests.
625    */
626   static class UselessTestAbortable implements Abortable {
627     boolean aborted = false;
628     @Override
629     public void abort(String why, Throwable e) {
630       LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
631       aborted = true;
632     }
633 
634     @Override
635     public boolean isAborted() {
636       return this.aborted;
637     }
638   }
639 
640   /**
641    * Verifies HBASE-5806.  When splitting is partially done and the master goes down
642    * when the SPLIT node is in either SPLIT or SPLITTING state.
643    *
644    * @throws IOException
645    * @throws InterruptedException
646    * @throws NodeExistsException
647    * @throws KeeperException
648    * @throws DeserializationException
649    */
650   @Test(timeout = 400000)
651   public void testMasterRestartWhenSplittingIsPartial()
652       throws IOException, InterruptedException, NodeExistsException,
653       KeeperException, DeserializationException, ServiceException {
654     final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
655 
656     if (!useZKForAssignment) {
657       // This test doesn't apply if not using ZK for assignment
658       return;
659     }
660 
661     // Create table then get the single region for our new table.
662     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
663     List<HRegion> regions = cluster.getRegions(tableName);
664     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
665 
666     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
667 
668     // Turn off balancer so it doesn't cut in and mess up our placements.
669     this.admin.setBalancerRunning(false, true);
670     // Turn off the meta scanner so it don't remove parent on us.
671     cluster.getMaster().setCatalogJanitorEnabled(false);
672     ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
673       "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
674     try {
675       // Add a bit of load up into the table so splittable.
676       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
677       // Get region pre-split.
678       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
679       printOutRegions(server, "Initial regions: ");
680       // Now, before we split, set special flag in master, a flag that has
681       // it FAIL the processing of split.
682       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
683       // Now try splitting and it should work.
684 
685       this.admin.split(hri.getRegionNameAsString());
686       checkAndGetDaughters(tableName);
687       // Assert the ephemeral node is up in zk.
688       String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
689       Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
690       LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
691           + stats);
692       byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
693       RegionTransition rtd = RegionTransition.parseFrom(bytes);
694       // State could be SPLIT or SPLITTING.
695       assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
696           || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
697 
698       // abort and wait for new master.
699       MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
700 
701       this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
702 
703       // Update the region to be offline and split, so that HRegionInfo#equals
704       // returns true in checking rebuilt region states map.
705       hri.setOffline(true);
706       hri.setSplit(true);
707       ServerName regionServerOfRegion = master.getAssignmentManager()
708         .getRegionStates().getRegionServerOfRegion(hri);
709       assertTrue(regionServerOfRegion != null);
710 
711       // Remove the block so that split can move ahead.
712       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
713       String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
714       Stat stat = new Stat();
715       byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
716       // ZKUtil.create
717       for (int i=0; data != null && i<60; i++) {
718         Thread.sleep(1000);
719         data = ZKUtil.getDataNoWatch(zkw, node, stat);
720       }
721       assertNull("Waited too long for ZK node to be removed: "+node, data);
722       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
723       assertTrue("Split parent should be in SPLIT state",
724         regionStates.isRegionInState(hri, State.SPLIT));
725       regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
726       assertTrue(regionServerOfRegion == null);
727     } finally {
728       // Set this flag back.
729       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
730       admin.setBalancerRunning(true, false);
731       cluster.getMaster().setCatalogJanitorEnabled(true);
732       t.close();
733       zkw.close();
734     }
735   }
736 
737   /**
738    * Verifies HBASE-5806.  Here the case is that splitting is completed but before the
739    * CJ could remove the parent region the master is killed and restarted.
740    * @throws IOException
741    * @throws InterruptedException
742    * @throws NodeExistsException
743    * @throws KeeperException
744    */
745   @Test (timeout = 300000)
746   public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
747       throws IOException, InterruptedException, NodeExistsException,
748       KeeperException, ServiceException {
749     final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
750 
751     // Create table then get the single region for our new table.
752     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
753     List<HRegion> regions = cluster.getRegions(tableName);
754     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
755 
756     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
757 
758     // Turn off balancer so it doesn't cut in and mess up our placements.
759     this.admin.setBalancerRunning(false, true);
760     // Turn off the meta scanner so it don't remove parent on us.
761     cluster.getMaster().setCatalogJanitorEnabled(false);
762     ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
763       "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
764     try {
765       // Add a bit of load up into the table so splittable.
766       TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
767       // Get region pre-split.
768       HRegionServer server = cluster.getRegionServer(tableRegionIndex);
769       printOutRegions(server, "Initial regions: ");
770 
771       this.admin.split(hri.getRegionNameAsString());
772       checkAndGetDaughters(tableName);
773       // Assert the ephemeral node is up in zk.
774       String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
775       Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
776       LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
777           + stats);
778       String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
779       Stat stat = new Stat();
780       byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
781       // ZKUtil.create
782       for (int i=0; data != null && i<60; i++) {
783         Thread.sleep(1000);
784         data = ZKUtil.getDataNoWatch(zkw, node, stat);
785       }
786       assertNull("Waited too long for ZK node to be removed: "+node, data);
787 
788       MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
789 
790       this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
791 
792       // Update the region to be offline and split, so that HRegionInfo#equals
793       // returns true in checking rebuilt region states map.
794       hri.setOffline(true);
795       hri.setSplit(true);
796       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
797       assertTrue("Split parent should be in SPLIT state",
798         regionStates.isRegionInState(hri, State.SPLIT));
799       ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
800       assertTrue(regionServerOfRegion == null);
801     } finally {
802       this.admin.setBalancerRunning(true, false);
803       cluster.getMaster().setCatalogJanitorEnabled(true);
804       t.close();
805       zkw.close();
806     }
807   }
808 
809   /**
810    *
811    * While transitioning node from RS_ZK_REGION_SPLITTING to
812    * RS_ZK_REGION_SPLITTING during region split,if zookeper went down split always
813    * fails for the region. HBASE-6088 fixes this scenario.
814    * This test case is to test the znode is deleted(if created) or not in roll back.
815    *
816    * @throws IOException
817    * @throws InterruptedException
818    * @throws KeeperException
819    */
820   @Test(timeout = 60000)
821   public void testSplitBeforeSettingSplittingInZK() throws Exception,
822       InterruptedException, KeeperException {
823     testSplitBeforeSettingSplittingInZKInternals();
824   }
825 
826   @Test(timeout = 60000)
827   public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
828     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
829     final TableName tableName =
830         TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
831     // Create table then get the single region for our new table.
832     HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
833     List<HRegion> regions = null;
834     try {
835       regions = cluster.getRegions(tableName);
836       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
837       HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
838       insertData(tableName.getName(), admin, t);
839       // Turn off balancer so it doesn't cut in and mess up our placements.
840       admin.setBalancerRunning(false, true);
841       // Turn off the meta scanner so it don't remove parent on us.
842       cluster.getMaster().setCatalogJanitorEnabled(false);
843       boolean tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
844           tableName);
845       assertEquals("The specified table should present.", true, tableExists);
846       final HRegion region = findSplittableRegion(regions);
847       assertTrue("not able to find a splittable region", region != null);
848       SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
849       try {
850         st.prepare();
851         st.createDaughters(regionServer, regionServer);
852       } catch (IOException e) {
853 
854       }
855       tableExists = MetaReader.tableExists(regionServer.getCatalogTracker(),
856           tableName);
857       assertEquals("The specified table should present.", true, tableExists);
858     } finally {
859       if (regions != null) {
860         String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
861             .getEncodedName());
862         ZKUtil.deleteNodeFailSilent(zkw, node);
863       }
864       admin.setBalancerRunning(true, false);
865       cluster.getMaster().setCatalogJanitorEnabled(true);
866       t.close();
867     }
868   }
869 
870   private void insertData(final byte[] tableName, HBaseAdmin admin, HTable t) throws IOException,
871       InterruptedException {
872     Put p = new Put(Bytes.toBytes("row1"));
873     p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
874     t.put(p);
875     p = new Put(Bytes.toBytes("row2"));
876     p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
877     t.put(p);
878     p = new Put(Bytes.toBytes("row3"));
879     p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
880     t.put(p);
881     p = new Put(Bytes.toBytes("row4"));
882     p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
883     t.put(p);
884     admin.flush(tableName);
885   }
886 
887   /**
888    * If a table has regions that have no store files in a region, they should split successfully
889    * into two regions with no store files.
890    */
891   @Test(timeout = 60000)
892   public void testSplitRegionWithNoStoreFiles()
893       throws Exception {
894     final TableName tableName =
895         TableName.valueOf("testSplitRegionWithNoStoreFiles");
896     // Create table then get the single region for our new table.
897     createTableAndWait(tableName.getName(), HConstants.CATALOG_FAMILY);
898     List<HRegion> regions = cluster.getRegions(tableName);
899     HRegionInfo hri = getAndCheckSingleTableRegion(regions);
900     ensureTableRegionNotOnSameServerAsMeta(admin, hri);
901     int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
902     HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
903     // Turn off balancer so it doesn't cut in and mess up our placements.
904     this.admin.setBalancerRunning(false, true);
905     // Turn off the meta scanner so it don't remove parent on us.
906     cluster.getMaster().setCatalogJanitorEnabled(false);
907     try {
908       // Precondition: we created a table with no data, no store files.
909       printOutRegions(regionServer, "Initial regions: ");
910       Configuration conf = cluster.getConfiguration();
911       HBaseFsck.debugLsr(conf, new Path("/"));
912       Path rootDir = FSUtils.getRootDir(conf);
913       FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
914       Map<String, Path> storefiles =
915           FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
916       assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
917 
918       // find a splittable region.  Refresh the regions list
919       regions = cluster.getRegions(tableName);
920       final HRegion region = findSplittableRegion(regions);
921       assertTrue("not able to find a splittable region", region != null);
922 
923       // Now split.
924       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
925       try {
926         st.prepare();
927         st.execute(regionServer, regionServer);
928       } catch (IOException e) {
929         fail("Split execution should have succeeded with no exceptions thrown");
930       }
931 
932       // Postcondition: split the table with no store files into two regions, but still have not
933       // store files
934       List<HRegion> daughters = cluster.getRegions(tableName);
935       assertTrue(daughters.size() == 2);
936 
937       // check dirs
938       HBaseFsck.debugLsr(conf, new Path("/"));
939       Map<String, Path> storefilesAfter =
940           FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
941       assertEquals("Expected nothing but found " + storefilesAfter.toString(),
942           storefilesAfter.size(), 0);
943 
944       hri = region.getRegionInfo(); // split parent
945       AssignmentManager am = cluster.getMaster().getAssignmentManager();
946       RegionStates regionStates = am.getRegionStates();
947       long start = EnvironmentEdgeManager.currentTimeMillis();
948       while (!regionStates.isRegionInState(hri, State.SPLIT)) {
949         assertFalse("Timed out in waiting split parent to be in state SPLIT",
950           EnvironmentEdgeManager.currentTimeMillis() - start > 60000);
951         Thread.sleep(500);
952       }
953 
954       // We should not be able to assign it again
955       am.assign(hri, true, true);
956       assertFalse("Split region can't be assigned",
957         regionStates.isRegionInTransition(hri));
958       assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
959 
960       // We should not be able to unassign it either
961       am.unassign(hri, true, null);
962       assertFalse("Split region can't be unassigned",
963         regionStates.isRegionInTransition(hri));
964       assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
965     } finally {
966       admin.setBalancerRunning(true, false);
967       cluster.getMaster().setCatalogJanitorEnabled(true);
968     }
969   }
970 
971   @Test(timeout = 180000)
972   public void testSplitHooksBeforeAndAfterPONR() throws Exception {
973     TableName firstTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1");
974     TableName secondTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2");
975     HColumnDescriptor hcd = new HColumnDescriptor("cf");
976 
977     HTableDescriptor desc = new HTableDescriptor(firstTable);
978     desc.addCoprocessor(MockedRegionObserver.class.getName());
979     desc.addFamily(hcd);
980     admin.createTable(desc);
981     TESTING_UTIL.waitUntilAllRegionsAssigned(firstTable);
982 
983     desc = new HTableDescriptor(secondTable);
984     desc.addFamily(hcd);
985     admin.createTable(desc);
986     TESTING_UTIL.waitUntilAllRegionsAssigned(secondTable);
987 
988     List<HRegion> firstTableRegions = cluster.getRegions(firstTable);
989     List<HRegion> secondTableRegions = cluster.getRegions(secondTable);
990 
991     // Check that both tables actually have regions.
992     if (firstTableRegions.size() == 0 || secondTableRegions.size() == 0) {
993       fail("Each table should have at least one region.");
994     }
995     ServerName serverName =
996         cluster.getServerHoldingRegion(firstTableRegions.get(0).getRegionName());
997     admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
998       Bytes.toBytes(serverName.getServerName()));
999     HTable table1 = null;
1000     HTable table2 = null;
1001     try {
1002       table1 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1003       table2 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1004       insertData(firstTable.getName(), admin, table1);
1005       insertData(secondTable.getName(), admin, table2);
1006       admin.split(firstTable.getName(), "row2".getBytes());
1007       firstTableRegions = cluster.getRegions(firstTable.getName());
1008       while (firstTableRegions.size() != 2) {
1009         Thread.sleep(1000);
1010         firstTableRegions = cluster.getRegions(firstTable.getName());
1011       }
1012       assertEquals("Number of regions after split should be 2.", 2, firstTableRegions.size());
1013       secondTableRegions = cluster.getRegions(secondTable.getName());
1014       assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
1015     } finally {
1016       if (table1 != null) {
1017         table1.close();
1018       }
1019       if (table2 != null) {
1020         table2.close();
1021       }
1022       TESTING_UTIL.deleteTable(firstTable);
1023       TESTING_UTIL.deleteTable(secondTable);
1024     }
1025   }
1026 
1027   private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
1028     final byte[] tableName = Bytes.toBytes("testSplitBeforeSettingSplittingInZK");
1029     try {
1030       // Create table then get the single region for our new table.
1031       createTableAndWait(tableName, Bytes.toBytes("cf"));
1032 
1033       List<HRegion> regions = awaitTableRegions(tableName);
1034       assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
1035 
1036       int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
1037       HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1038       final HRegion region = findSplittableRegion(regions);
1039       assertTrue("not able to find a splittable region", region != null);
1040       SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
1041         @Override
1042         public PairOfSameType<HRegion> stepsBeforePONR(final Server server,
1043             final RegionServerServices services, boolean testing) throws IOException {
1044           throw new SplittingNodeCreationFailedException ();
1045         }
1046       };
1047       String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1048           region.getRegionInfo().getEncodedName());
1049       regionServer.getZooKeeper().sync(node);
1050       for (int i = 0; i < 100; i++) {
1051         // We expect the znode to be deleted by this time. Here the
1052         // znode could be in OPENED state and the
1053         // master has not yet deleted the znode.
1054         if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
1055           Thread.sleep(100);
1056         }
1057       }
1058       try {
1059         st.prepare();
1060         st.execute(regionServer, regionServer);
1061       } catch (IOException e) {
1062         // check for the specific instance in case the Split failed due to the
1063         // existence of the znode in OPENED state.
1064         // This will at least make the test to fail;
1065         assertTrue("Should be instance of CreateSplittingNodeFailedException",
1066             e instanceof SplittingNodeCreationFailedException );
1067         node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1068             region.getRegionInfo().getEncodedName());
1069         {
1070           assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1071         }
1072         assertTrue(st.rollback(regionServer, regionServer));
1073         assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1074       }
1075     } finally {
1076       TESTING_UTIL.deleteTable(tableName);
1077     }
1078   }
1079 
1080   @Test
1081   public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
1082       throws Exception {
1083     final TableName tableName =
1084         TableName.valueOf("testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck");
1085     try {
1086       HTableDescriptor htd = new HTableDescriptor(tableName);
1087       htd.addFamily(new HColumnDescriptor("f"));
1088       htd.setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName());
1089       admin.createTable(htd);
1090       List<HRegion> regions = awaitTableRegions(tableName.toBytes());
1091       HRegion region = regions.get(0);
1092       for(int i = 3;i<9;i++) {
1093         Put p = new Put(Bytes.toBytes("row"+i));
1094         p.add(Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("value"+i));
1095         region.put(p);
1096       }
1097       region.flushcache();
1098       Store store = region.getStore(Bytes.toBytes("f"));
1099       Collection<StoreFile> storefiles = store.getStorefiles();
1100       assertEquals(storefiles.size(), 1);
1101       assertFalse(region.hasReferences());
1102       Path referencePath = region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
1103         storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
1104       assertNotNull(referencePath);
1105     } finally {
1106       TESTING_UTIL.deleteTable(tableName);
1107     }
1108   }
1109 
1110   public static class MockedSplitTransaction extends SplitTransaction {
1111 
1112     private HRegion currentRegion;
1113     public MockedSplitTransaction(HRegion r, byte[] splitrow) {
1114       super(r, splitrow);
1115       this.currentRegion = r;
1116     }
1117 
1118     @Override
1119     void transitionZKNode(Server server, RegionServerServices services, HRegion a, HRegion b)
1120         throws IOException {
1121       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1122           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1123         try {
1124           if (!secondSplit){
1125             callRollBack = true;
1126             latch.await();
1127           }
1128         } catch (InterruptedException e) {
1129         }
1130 
1131       }
1132       super.transitionZKNode(server, services, a, b);
1133       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1134           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1135         firstSplitCompleted = true;
1136       }
1137     }
1138     @Override
1139     public boolean rollback(Server server, RegionServerServices services) throws IOException {
1140       if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1141           .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1142         if(secondSplit){
1143           super.rollback(server, services);
1144           latch.countDown();
1145           return true;
1146         }
1147       }
1148       return super.rollback(server, services);
1149     }
1150 
1151   }
1152 
1153   private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
1154     for (int i = 0; i < 5; ++i) {
1155       for (HRegion r: regions) {
1156         if (r.isSplittable()) {
1157           return(r);
1158         }
1159       }
1160       Thread.sleep(100);
1161     }
1162     return(null);
1163   }
1164 
1165   @Test
1166   public void testFailedSplit() throws Exception {
1167     TableName tableName = TableName.valueOf("testFailedSplit");
1168     byte[] colFamily = Bytes.toBytes("info");
1169     TESTING_UTIL.createTable(tableName, colFamily);
1170     HTable table = new HTable(TESTING_UTIL.getConfiguration(), tableName);
1171     try {
1172       TESTING_UTIL.loadTable(table, colFamily);
1173       List<HRegionInfo> regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1174       assertTrue(regions.size() == 1);
1175       final HRegion actualRegion = cluster.getRegions(tableName).get(0);
1176       actualRegion.getCoprocessorHost().load(FailingSplitRegionObserver.class,
1177         Coprocessor.PRIORITY_USER, actualRegion.getBaseConf());
1178 
1179       // The following split would fail.
1180       admin.split(tableName.getNameAsString());
1181       FailingSplitRegionObserver.latch.await();
1182       LOG.info("Waiting for region to come out of RIT");
1183       TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1184         @Override
1185         public boolean evaluate() throws Exception {
1186           RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
1187           Map<String, RegionState> rit = regionStates.getRegionsInTransition();
1188           return !rit.containsKey(actualRegion.getRegionInfo().getEncodedName());
1189         }
1190       });
1191       regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1192       assertTrue(regions.size() == 1);
1193     } finally {
1194       table.close();
1195       TESTING_UTIL.deleteTable(tableName);
1196     }
1197   }
1198 
1199   private List<HRegion> checkAndGetDaughters(byte[] tableName)
1200       throws InterruptedException {
1201     List<HRegion> daughters = null;
1202     // try up to 10s
1203     for (int i=0; i<100; i++) {
1204       daughters = cluster.getRegions(tableName);
1205       if (daughters.size() >= 2) break;
1206       Thread.sleep(100);
1207     }
1208     assertTrue(daughters.size() >= 2);
1209     return daughters;
1210   }
1211 
1212   private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
1213   throws IOException, InterruptedException {
1214     cluster.abortMaster(0);
1215     cluster.waitOnMaster(0);
1216     cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
1217     		MockMasterWithoutCatalogJanitor.class, HMaster.class);
1218     MockMasterWithoutCatalogJanitor master = null;
1219     master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
1220     cluster.waitForActiveAndReadyMaster();
1221     return master;
1222   }
1223 
1224   private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
1225       throws IOException, InterruptedException {
1226     this.admin.split(hri.getRegionNameAsString());
1227     try {
1228       for (int i = 0; ProtobufUtil.getOnlineRegions(server).size() <= regionCount && i < 300; i++) {
1229         LOG.debug("Waiting on region to split");
1230         Thread.sleep(100);
1231       }
1232 
1233       assertFalse("Waited too long for split",
1234         ProtobufUtil.getOnlineRegions(server).size() <= regionCount);
1235     } catch (RegionServerStoppedException e) {
1236       if (useZKForAssignment) {
1237         // If not using ZK for assignment, the exception may be expected.
1238         LOG.error(e);
1239         throw e;
1240       }
1241     }
1242   }
1243 
1244   /**
1245    * Ensure single table region is not on same server as the single hbase:meta table
1246    * region.
1247    * @param admin
1248    * @param hri
1249    * @return Index of the server hosting the single table region
1250    * @throws UnknownRegionException
1251    * @throws MasterNotRunningException
1252    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
1253    * @throws InterruptedException
1254    */
1255   private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin,
1256       final HRegionInfo hri)
1257   throws HBaseIOException, MasterNotRunningException,
1258   ZooKeeperConnectionException, InterruptedException {
1259     // Now make sure that the table region is not on same server as that hosting
1260     // hbase:meta  We don't want hbase:meta replay polluting our test when we later crash
1261     // the table region serving server.
1262     int metaServerIndex = cluster.getServerWithMeta();
1263     assertTrue(metaServerIndex != -1);
1264     HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1265     int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1266     assertTrue(tableRegionIndex != -1);
1267     HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1268     if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1269       HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1270       assertNotNull(hrs);
1271       assertNotNull(hri);
1272       LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1273         metaRegionServer.getServerName() + " to " +
1274         hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1275       admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1276     }
1277     // Wait till table region is up on the server that is NOT carrying hbase:meta.
1278     for (int i = 0; i < 100; i++) {
1279       tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1280       if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1281       LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1282         tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1283       Thread.sleep(100);
1284     }
1285     assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1286         && tableRegionIndex != metaServerIndex);
1287     // Verify for sure table region is not on same server as hbase:meta
1288     tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1289     assertTrue(tableRegionIndex != -1);
1290     assertNotSame(metaServerIndex, tableRegionIndex);
1291     return tableRegionIndex;
1292   }
1293 
1294   /**
1295    * Find regionserver other than the one passed.
1296    * Can't rely on indexes into list of regionservers since crashed servers
1297    * occupy an index.
1298    * @param cluster
1299    * @param notThisOne
1300    * @return A regionserver that is not <code>notThisOne</code> or null if none
1301    * found
1302    */
1303   private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1304       final HRegionServer notThisOne) {
1305     for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1306       HRegionServer hrs = rst.getRegionServer();
1307       if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1308       if (hrs.isStopping() || hrs.isStopped()) continue;
1309       return hrs;
1310     }
1311     return null;
1312   }
1313 
1314   private void printOutRegions(final HRegionServer hrs, final String prefix)
1315       throws IOException {
1316     List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
1317     for (HRegionInfo region: regions) {
1318       LOG.info(prefix + region.getRegionNameAsString());
1319     }
1320   }
1321 
1322   private void waitUntilRegionServerDead() throws InterruptedException {
1323     // Wait until the master processes the RS shutdown
1324     for (int i=0; cluster.getMaster().getClusterStatus().
1325         getServers().size() == NB_SERVERS && i<100; i++) {
1326       LOG.info("Waiting on server to go down");
1327       Thread.sleep(100);
1328     }
1329     assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1330         getServers().size() == NB_SERVERS);
1331   }
1332 
1333   private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
1334     // Wait till regions are back on line again.
1335     for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1336       LOG.info("Waiting for repair to happen");
1337       Thread.sleep(1000);
1338     }
1339     if (cluster.getRegions(tableName).size() < numDaughters) {
1340       fail("Waiting too long for daughter regions");
1341     }
1342   }
1343 
1344   private List<HRegion> awaitTableRegions(final byte[] tableName) throws InterruptedException {
1345     List<HRegion> regions = null;
1346     for (int i = 0; i < 100; i++) {
1347       regions = cluster.getRegions(tableName);
1348       if (regions.size() > 0) break;
1349       Thread.sleep(100);
1350     }
1351     return regions;
1352   }
1353 
1354   private HTable createTableAndWait(byte[] tableName, byte[] cf) throws IOException,
1355       InterruptedException {
1356     HTable t = TESTING_UTIL.createTable(tableName, cf);
1357     awaitTableRegions(tableName);
1358     assertTrue("Table not online: " + Bytes.toString(tableName),
1359       cluster.getRegions(tableName).size() != 0);
1360     return t;
1361   }
1362 
1363   public static class MockMasterWithoutCatalogJanitor extends HMaster {
1364 
1365     public MockMasterWithoutCatalogJanitor(Configuration conf) throws IOException, KeeperException,
1366         InterruptedException {
1367       super(conf);
1368     }
1369   }
1370 
1371   private static class SplittingNodeCreationFailedException  extends IOException {
1372     private static final long serialVersionUID = 1652404976265623004L;
1373 
1374     public SplittingNodeCreationFailedException () {
1375       super();
1376     }
1377   }
1378 
1379   public static class MockedRegionObserver extends BaseRegionObserver {
1380     private SplitTransaction st = null;
1381     private PairOfSameType<HRegion> daughterRegions = null;
1382 
1383     @Override
1384     public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
1385         byte[] splitKey, List<Mutation> metaEntries) throws IOException {
1386       RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1387       HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1388       List<HRegion> onlineRegions =
1389           rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
1390       HRegion region = onlineRegions.get(0);
1391       for (HRegion r : onlineRegions) {
1392         if (r.getRegionInfo().containsRow(splitKey)) {
1393           region = r;
1394           break;
1395         }
1396       }
1397       st = new SplitTransaction(region, splitKey);
1398       if (!st.prepare()) {
1399         LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
1400             + " failed. So returning null. ");
1401         ctx.bypass();
1402         return;
1403       }
1404       region.forceSplit(splitKey);
1405       daughterRegions = st.stepsBeforePONR(rs, rs, false);
1406       HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
1407       copyOfParent.setOffline(true);
1408       copyOfParent.setSplit(true);
1409       // Put for parent
1410       Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
1411       MetaEditor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
1412         daughterRegions.getSecond().getRegionInfo());
1413       metaEntries.add(putParent);
1414       // Puts for daughters
1415       Put putA = MetaEditor.makePutFromRegionInfo(daughterRegions.getFirst().getRegionInfo());
1416       Put putB = MetaEditor.makePutFromRegionInfo(daughterRegions.getSecond().getRegionInfo());
1417       st.addLocation(putA, rs.getServerName(), 1);
1418       st.addLocation(putB, rs.getServerName(), 1);
1419       metaEntries.add(putA);
1420       metaEntries.add(putB);
1421     }
1422 
1423     @Override
1424     public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
1425         throws IOException {
1426       RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1427       HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1428       st.stepsAfterPONR(rs, rs, daughterRegions);
1429     }
1430 
1431   }
1432 
1433   static class CustomSplitPolicy extends RegionSplitPolicy {
1434 
1435     @Override
1436     protected boolean shouldSplit() {
1437       return true;
1438     }
1439 
1440     @Override
1441     public boolean skipStoreFileRangeCheck() {
1442       return true;
1443     }
1444   }
1445 }
1446