1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertArrayEquals;
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.NavigableMap;
29  import java.util.Random;
30  import java.util.Set;
31  
32  import org.apache.commons.io.IOUtils;
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.Chore;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HServerAddress;
41  import org.apache.hadoop.hbase.LargeTests;
42  import org.apache.hadoop.hbase.NotServingRegionException;
43  import org.apache.hadoop.hbase.ServerName;
44  import org.apache.hadoop.hbase.Stoppable;
45  import org.apache.hadoop.hbase.catalog.MetaEditor;
46  import org.apache.hadoop.hbase.catalog.MetaReader;
47  import org.apache.hadoop.hbase.client.Get;
48  import org.apache.hadoop.hbase.client.HBaseAdmin;
49  import org.apache.hadoop.hbase.client.HConnection;
50  import org.apache.hadoop.hbase.client.HConnectionManager;
51  import org.apache.hadoop.hbase.client.HTable;
52  import org.apache.hadoop.hbase.client.MetaScanner;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.client.Result;
55  import org.apache.hadoop.hbase.client.Scan;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.util.Pair;
58  import org.apache.hadoop.hbase.util.PairOfSameType;
59  import org.apache.hadoop.hbase.util.StoppableImplementation;
60  import org.apache.hadoop.hbase.util.Threads;
61  import org.junit.AfterClass;
62  import org.junit.BeforeClass;
63  import org.junit.Test;
64  import org.junit.experimental.categories.Category;
65  
66  import com.google.common.collect.Iterators;
67  import com.google.common.collect.Sets;
68  
69  @Category(LargeTests.class)
70  public class TestEndToEndSplitTransaction {
71    private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class);
72    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
73    private static final Configuration conf = TEST_UTIL.getConfiguration();
74  
75    @BeforeClass
76    public static void beforeAllTests() throws Exception {
77      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
78      TEST_UTIL.startMiniCluster();
79    }
80  
81    @AfterClass
82    public static void afterAllTests() throws Exception {
83      TEST_UTIL.shutdownMiniCluster();
84    }
85  
86    @Test
87    public void testMasterOpsWhileSplitting() throws Exception {
88      byte[] tableName = Bytes.toBytes("TestSplit");
89      byte[] familyName = Bytes.toBytes("fam");
90      HTable ht = TEST_UTIL.createTable(tableName, familyName);
91      TEST_UTIL.loadTable(ht, familyName);
92      ht.close();
93      HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
94      byte []firstRow = Bytes.toBytes("aaa");
95      byte []splitRow = Bytes.toBytes("lll");
96      byte []lastRow = Bytes.toBytes("zzz");
97      HConnection con = HConnectionManager
98          .getConnection(TEST_UTIL.getConfiguration());
99      // this will also cache the region
100     byte[] regionName = con.locateRegion(tableName, splitRow).getRegionInfo()
101         .getRegionName();
102     HRegion region = server.getRegion(regionName);
103     SplitTransaction split = new SplitTransaction(region, splitRow);
104     split.prepare();
105 
106     // 1. phase I
107     PairOfSameType<HRegion> regions = split.createDaughters(server, server);
108     assertFalse(test(con, tableName, firstRow, server));
109     assertFalse(test(con, tableName, lastRow, server));
110 
111     // passing null as services prevents final step
112     // 2, most of phase II
113     split.openDaughters(server, null, regions.getFirst(), regions.getSecond());
114     assertFalse(test(con, tableName, firstRow, server));
115     assertFalse(test(con, tableName, lastRow, server));
116 
117     // 3. finish phase II
118     // note that this replicates some code from SplitTransaction
119     // 2nd daughter first
120     server.postOpenDeployTasks(regions.getSecond(), server.getCatalogTracker(), true);
121     // Add to online regions
122     server.addToOnlineRegions(regions.getSecond());
123     // THIS is the crucial point:
124     // the 2nd daughter was added, so querying before the split key should fail.
125     assertFalse(test(con, tableName, firstRow, server));
126     // past splitkey is ok.
127     assertTrue(test(con, tableName, lastRow, server));
128 
129     // first daughter second
130     server.postOpenDeployTasks(regions.getFirst(), server.getCatalogTracker(), true);
131     // Add to online regions
132     server.addToOnlineRegions(regions.getFirst());
133     assertTrue(test(con, tableName, firstRow, server));
134     assertTrue(test(con, tableName, lastRow, server));
135 
136     // 4. phase III
137     split.transitionZKNode(server, server, regions.getFirst(),
138         regions.getSecond());
139     assertTrue(test(con, tableName, firstRow, server));
140     assertTrue(test(con, tableName, lastRow, server));
141   }
142 
143   /**
144    * attempt to locate the region and perform a get and scan
145    * @return True if successful, False otherwise.
146    */
147   private boolean test(HConnection con, byte[] tableName, byte[] row,
148       HRegionServer server) {
149     // not using HTable to avoid timeouts and retries
150     try {
151       byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo()
152           .getRegionName();
153       // get and scan should now succeed without exception
154       server.get(regionName, new Get(row));
155       server.openScanner(regionName, new Scan(row));
156     } catch (IOException x) {
157       return false;
158     }
159     return true;
160   }
161 
162   /**
163    * Tests that the client sees meta table changes as atomic during splits
164    */
165   @Test
166   public void testFromClientSideWhileSplitting() throws Throwable {
167     LOG.info("Starting testFromClientSideWhileSplitting");
168     final byte[] TABLENAME = Bytes.toBytes("testFromClientSideWhileSplitting");
169     final byte[] FAMILY = Bytes.toBytes("family");
170 
171     //SplitTransaction will update the meta table by offlining the parent region, and adding info
172     //for daughters.
173     HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
174 
175     Stoppable stopper = new StoppableImplementation();
176     RegionSplitter regionSplitter = new RegionSplitter(table);
177     RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);
178 
179     regionChecker.start();
180     regionSplitter.start();
181 
182     //wait until the splitter is finished
183     regionSplitter.join();
184     stopper.stop(null);
185 
186     if (regionChecker.ex != null) {
187       throw regionChecker.ex;
188     }
189 
190     if (regionSplitter.ex != null) {
191       throw regionSplitter.ex;
192     }
193 
194     //one final check
195     regionChecker.verify();
196   }
197 
198   static class RegionSplitter extends Thread {
199     Throwable ex;
200     HTable table;
201     byte[] tableName, family;
202     HBaseAdmin admin;
203     HRegionServer rs;
204 
205     RegionSplitter(HTable table) throws IOException {
206       this.table = table;
207       this.tableName = table.getTableName();
208       this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
209       admin = TEST_UTIL.getHBaseAdmin();
210       rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
211     }
212 
213     public void run() {
214       try {
215         Random random = new Random();
216         for (int i=0; i< 5; i++) {
217           NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName, false);
218           if (regions.size() == 0) {
219             continue;
220           }
221           int regionIndex = random.nextInt(regions.size());
222 
223           //pick a random region and split it into two
224           HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex);
225 
226           //pick the mid split point
227           int start = 0, end = Integer.MAX_VALUE;
228           if (region.getStartKey().length > 0) {
229             start = Bytes.toInt(region.getStartKey());
230           }
231           if (region.getEndKey().length > 0) {
232             end = Bytes.toInt(region.getEndKey());
233           }
234           int mid = start + ((end - start) / 2);
235           byte[] splitPoint = Bytes.toBytes(mid);
236 
237           //put some rows to the regions
238           addData(start);
239           addData(mid);
240 
241           flushAndBlockUntilDone(admin, rs, region.getRegionName());
242           compactAndBlockUntilDone(admin, rs, region.getRegionName());
243 
244           log("Initiating region split for:" + region.getRegionNameAsString());
245           try {
246             admin.split(region.getRegionName(), splitPoint);
247             //wait until the split is complete
248             blockUntilRegionSplit(conf, 50000, region.getRegionName(), true);
249 
250           } catch (NotServingRegionException ex) {
251             //ignore
252           }
253         }
254       } catch (Throwable ex) {
255         this.ex = ex;
256       }
257     }
258 
259     void addData(int start) throws IOException {
260       for (int i=start; i< start + 100; i++) {
261         Put put = new Put(Bytes.toBytes(i));
262 
263         put.add(family, family, Bytes.toBytes(i));
264         table.put(put);
265       }
266       table.flushCommits();
267     }
268   }
269 
270   /**
271    * Checks regions using MetaScanner, MetaReader and HTable methods
272    */
273   static class RegionChecker extends Chore {
274     Configuration conf;
275     byte[] tableName;
276     Throwable ex;
277 
278     RegionChecker(Configuration conf, Stoppable stopper, byte[] tableName) {
279       super("RegionChecker", 10, stopper);
280       this.conf = conf;
281       this.tableName = tableName;
282       this.setDaemon(true);
283     }
284 
285     /** verify region boundaries obtained from MetaScanner */
286     void verifyRegionsUsingMetaScanner() throws Exception {
287 
288       //MetaScanner.allTableRegions()
289       NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName,
290           false);
291       verifyTableRegions(regions.keySet());
292 
293       //MetaScanner.listAllRegions()
294       List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, false);
295       verifyTableRegions(Sets.newTreeSet(regionList));
296     }
297 
298     /** verify region boundaries obtained from HTable.getStartEndKeys() */
299     void verifyRegionsUsingHTable() throws IOException {
300       HTable table = null;
301       try {
302         //HTable.getStartEndKeys()
303         table = new HTable(conf, tableName);
304         Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
305         verifyStartEndKeys(keys);
306 
307         //HTable.getRegionsInfo()
308         Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
309         verifyTableRegions(regions.keySet());
310       } finally {
311         IOUtils.closeQuietly(table);
312       }
313     }
314 
315     void verify() throws Exception {
316       verifyRegionsUsingMetaScanner();
317       verifyRegionsUsingHTable();
318     }
319 
320     void verifyTableRegions(Set<HRegionInfo> regions) {
321       log("Verifying " + regions.size() + " regions");
322 
323       byte[][] startKeys = new byte[regions.size()][];
324       byte[][] endKeys = new byte[regions.size()][];
325 
326       int i=0;
327       for (HRegionInfo region : regions) {
328         startKeys[i] = region.getStartKey();
329         endKeys[i] = region.getEndKey();
330         i++;
331       }
332 
333       Pair<byte[][], byte[][]> keys = new Pair<byte[][], byte[][]>(startKeys, endKeys);
334       verifyStartEndKeys(keys);
335     }
336 
337     void verifyStartEndKeys(Pair<byte[][], byte[][]> keys) {
338       byte[][] startKeys = keys.getFirst();
339       byte[][] endKeys = keys.getSecond();
340       assertEquals(startKeys.length, endKeys.length);
341       assertTrue("Found 0 regions for the table", startKeys.length > 0);
342 
343       assertArrayEquals("Start key for the first region is not byte[0]",
344           HConstants.EMPTY_START_ROW, startKeys[0]);
345       byte[] prevEndKey = HConstants.EMPTY_START_ROW;
346 
347       // ensure that we do not have any gaps
348       for (int i=0; i<startKeys.length; i++) {
349         assertArrayEquals(
350             "Hole in .META. is detected. prevEndKey=" + Bytes.toStringBinary(prevEndKey)
351                 + " ,regionStartKey=" + Bytes.toStringBinary(startKeys[i]), prevEndKey,
352             startKeys[i]);
353         prevEndKey = endKeys[i];
354       }
355       assertArrayEquals("End key for the last region is not byte[0]", HConstants.EMPTY_END_ROW,
356           endKeys[endKeys.length - 1]);
357     }
358 
359     @Override
360     protected void chore() {
361       try {
362         verify();
363       } catch (Throwable ex) {
364         this.ex = ex;
365         stopper.stop("caught exception");
366       }
367     }
368   }
369 
370   public static void log(String msg) {
371     LOG.info(msg);
372   }
373 
374   /* some utility methods for split tests */
375 
376   public static void flushAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
377       throws IOException, InterruptedException {
378     log("flushing region: " + Bytes.toStringBinary(regionName));
379     admin.flush(regionName);
380     log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
381     Threads.sleepWithoutInterrupt(500);
382     while (rs.cacheFlusher.getFlushQueueSize() > 0) {
383       Threads.sleep(50);
384     }
385   }
386 
387   public static void compactAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
388       throws IOException, InterruptedException {
389     log("Compacting region: " + Bytes.toStringBinary(regionName));
390     admin.majorCompact(regionName);
391     log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
392     Threads.sleepWithoutInterrupt(500);
393     while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
394       Threads.sleep(50);
395     }
396   }
397 
398   /** Blocks until the region split is complete in META and region server opens the daughters */
399   public static void blockUntilRegionSplit(Configuration conf, long timeout,
400       final byte[] regionName, boolean waitForDaughters)
401       throws IOException, InterruptedException {
402     long start = System.currentTimeMillis();
403     log("blocking until region is split:" +  Bytes.toStringBinary(regionName));
404     HRegionInfo daughterA = null, daughterB = null;
405     HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
406 
407     try {
408       while (System.currentTimeMillis() - start < timeout) {
409         Result result = getRegionRow(metaTable, regionName);
410         if (result == null) {
411           break;
412         }
413 
414         HRegionInfo region = MetaReader.parseCatalogResult(result).getFirst();
415         if(region.isSplitParent()) {
416           log("found parent region: " + region.toString());
417           PairOfSameType<HRegionInfo> pair = MetaReader.getDaughterRegions(result);
418           daughterA = pair.getFirst();
419           daughterB = pair.getSecond();
420           break;
421         }
422         Threads.sleep(100);
423       }
424 
425       //if we are here, this means the region split is complete or timed out
426       if (waitForDaughters) {
427         long rem = timeout - (System.currentTimeMillis() - start);
428         blockUntilRegionIsInMeta(metaTable, rem, daughterA);
429 
430         rem = timeout - (System.currentTimeMillis() - start);
431         blockUntilRegionIsInMeta(metaTable, rem, daughterB);
432 
433         rem = timeout - (System.currentTimeMillis() - start);
434         blockUntilRegionIsOpened(conf, rem, daughterA);
435 
436         rem = timeout - (System.currentTimeMillis() - start);
437         blockUntilRegionIsOpened(conf, rem, daughterB);
438       }
439     } finally {
440       IOUtils.closeQuietly(metaTable);
441     }
442   }
443 
444   public static Result getRegionRow(HTable metaTable, byte[] regionName) throws IOException {
445     Get get = new Get(regionName);
446     return metaTable.get(get);
447   }
448 
449   public static void blockUntilRegionIsInMeta(HTable metaTable, long timeout, HRegionInfo hri)
450       throws IOException, InterruptedException {
451     log("blocking until region is in META: " + hri.getRegionNameAsString());
452     long start = System.currentTimeMillis();
453     while (System.currentTimeMillis() - start < timeout) {
454       Result result = getRegionRow(metaTable, hri.getRegionName());
455       if (result != null) {
456         HRegionInfo info = MetaReader.parseCatalogResult(result).getFirst();
457         if (info != null && !info.isOffline()) {
458           log("found region in META: " + hri.getRegionNameAsString());
459           break;
460         }
461       }
462       Threads.sleep(10);
463     }
464   }
465 
466   public static void blockUntilRegionIsOpened(Configuration conf, long timeout, HRegionInfo hri)
467       throws IOException, InterruptedException {
468     log("blocking until region is opened for reading:" + hri.getRegionNameAsString());
469     long start = System.currentTimeMillis();
470     HTable table = new HTable(conf, hri.getTableName());
471 
472     try {
473       Get get = new Get(hri.getStartKey());
474       while (System.currentTimeMillis() - start < timeout) {
475         try {
476           table.get(get);
477           break;
478         } catch(IOException ex) {
479           //wait some more
480         }
481         Threads.sleep(10);
482       }
483     } finally {
484       IOUtils.closeQuietly(table);
485     }
486   }
487 
488   @org.junit.Rule
489   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
490     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
491 }
492