1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertArrayEquals;
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.NavigableMap;
29  import java.util.Random;
30  import java.util.Set;
31  
32  import org.apache.commons.io.IOUtils;
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.Chore;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HServerAddress;
41  import org.apache.hadoop.hbase.LargeTests;
42  import org.apache.hadoop.hbase.NotServingRegionException;
43  import org.apache.hadoop.hbase.ServerName;
44  import org.apache.hadoop.hbase.Stoppable;
45  import org.apache.hadoop.hbase.catalog.MetaEditor;
46  import org.apache.hadoop.hbase.catalog.MetaReader;
47  import org.apache.hadoop.hbase.client.Get;
48  import org.apache.hadoop.hbase.client.HBaseAdmin;
49  import org.apache.hadoop.hbase.client.HConnection;
50  import org.apache.hadoop.hbase.client.HConnectionManager;
51  import org.apache.hadoop.hbase.client.HTable;
52  import org.apache.hadoop.hbase.client.MetaScanner;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.client.Result;
55  import org.apache.hadoop.hbase.client.Scan;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.util.Pair;
58  import org.apache.hadoop.hbase.util.PairOfSameType;
59  import org.apache.hadoop.hbase.util.StoppableImplementation;
60  import org.apache.hadoop.hbase.util.Threads;
61  import org.junit.AfterClass;
62  import org.junit.BeforeClass;
63  import org.junit.Test;
64  import org.junit.experimental.categories.Category;
65  
66  import com.google.common.collect.Iterators;
67  import com.google.common.collect.Sets;
68  
69  @Category(LargeTests.class)
70  public class TestEndToEndSplitTransaction {
71    private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class);
72    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
73    private static final Configuration conf = TEST_UTIL.getConfiguration();
74  
75    @BeforeClass
76    public static void beforeAllTests() throws Exception {
77      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
78      TEST_UTIL.startMiniCluster();
79    }
80  
81    @AfterClass
82    public static void afterAllTests() throws Exception {
83      TEST_UTIL.shutdownMiniCluster();
84    }
85  
86    @Test
87    public void testMasterOpsWhileSplitting() throws Exception {
88      byte[] tableName = Bytes.toBytes("TestSplit");
89      byte[] familyName = Bytes.toBytes("fam");
90      HTable ht = TEST_UTIL.createTable(tableName, familyName);
91      TEST_UTIL.loadTable(ht, familyName);
92      ht.close();
93      HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
94      byte []firstRow = Bytes.toBytes("aaa");
95      byte []splitRow = Bytes.toBytes("lll");
96      byte []lastRow = Bytes.toBytes("zzz");
97      HConnection con = HConnectionManager
98          .getConnection(TEST_UTIL.getConfiguration());
99      // this will also cache the region
100     byte[] regionName = con.locateRegion(tableName, splitRow).getRegionInfo()
101         .getRegionName();
102     HRegion region = server.getRegion(regionName);
103     SplitTransaction split = new SplitTransaction(region, splitRow);
104     split.prepare();
105 
106     // 1. phase I
107     PairOfSameType<HRegion> regions = split.createDaughters(server, server);
108     assertFalse(test(con, tableName, firstRow, server));
109     assertFalse(test(con, tableName, lastRow, server));
110 
111     // passing null as services prevents final step
112     // 2, most of phase II
113     split.openDaughters(server, null, regions.getFirst(), regions.getSecond());
114     assertFalse(test(con, tableName, firstRow, server));
115     assertFalse(test(con, tableName, lastRow, server));
116 
117     // 3. finish phase II
118     // note that this replicates some code from SplitTransaction
119     // 2nd daughter first
120     server.postOpenDeployTasks(regions.getSecond(), server.getCatalogTracker(), true);
121     // Add to online regions
122     server.addToOnlineRegions(regions.getSecond());
123     // THIS is the crucial point:
124     // the 2nd daughter was added, so querying before the split key should fail.
125     assertFalse(test(con, tableName, firstRow, server));
126     // past splitkey is ok.
127     assertTrue(test(con, tableName, lastRow, server));
128 
129     // first daughter second
130     server.postOpenDeployTasks(regions.getFirst(), server.getCatalogTracker(), true);
131     // Add to online regions
132     server.addToOnlineRegions(regions.getFirst());
133     assertTrue(test(con, tableName, firstRow, server));
134     assertTrue(test(con, tableName, lastRow, server));
135 
136     // 4. phase III
137     split.transitionZKNode(server, server, regions.getFirst(),
138         regions.getSecond());
139     assertTrue(test(con, tableName, firstRow, server));
140     assertTrue(test(con, tableName, lastRow, server));
141   }
142 
143   /**
144    * attempt to locate the region and perform a get and scan
145    * @return True if successful, False otherwise.
146    */
147   private boolean test(HConnection con, byte[] tableName, byte[] row,
148       HRegionServer server) {
149     // not using HTable to avoid timeouts and retries
150     try {
151       byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo()
152           .getRegionName();
153       // get and scan should now succeed without exception
154       server.get(regionName, new Get(row));
155       server.openScanner(regionName, new Scan(row));
156     } catch (IOException x) {
157       return false;
158     }
159     return true;
160   }
161 
162   /**
163    * Tests that the client sees meta table changes as atomic during splits
164    */
165   @Test
166   public void testFromClientSideWhileSplitting() throws Throwable {
167     LOG.info("Starting testFromClientSideWhileSplitting");
168     final byte[] TABLENAME = Bytes.toBytes("testFromClientSideWhileSplitting");
169     final byte[] FAMILY = Bytes.toBytes("family");
170 
171     //SplitTransaction will update the meta table by offlining the parent region, and adding info
172     //for daughters.
173     HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY);
174 
175     Stoppable stopper = new StoppableImplementation();
176     RegionSplitter regionSplitter = new RegionSplitter(table);
177     RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);
178 
179     regionChecker.start();
180     regionSplitter.start();
181 
182     //wait until the splitter is finished
183     regionSplitter.join();
184     stopper.stop(null);
185 
186     if (regionChecker.ex != null) {
187       throw regionChecker.ex;
188     }
189 
190     if (regionSplitter.ex != null) {
191       throw regionSplitter.ex;
192     }
193 
194     //one final check
195     regionChecker.verify();
196   }
197 
198   static class RegionSplitter extends Thread {
199     Throwable ex;
200     HTable table;
201     byte[] tableName, family;
202     HBaseAdmin admin;
203     HRegionServer rs;
204 
205     RegionSplitter(HTable table) throws IOException {
206       this.table = table;
207       this.tableName = table.getTableName();
208       this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
209       admin = TEST_UTIL.getHBaseAdmin();
210       rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
211     }
212 
213     public void run() {
214       try {
215         Random random = new Random();
216         for (int i=0; i< 5; i++) {
217           NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, null,
218               tableName, false);
219           if (regions.size() == 0) {
220             continue;
221           }
222           int regionIndex = random.nextInt(regions.size());
223 
224           //pick a random region and split it into two
225           HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex);
226 
227           //pick the mid split point
228           int start = 0, end = Integer.MAX_VALUE;
229           if (region.getStartKey().length > 0) {
230             start = Bytes.toInt(region.getStartKey());
231           }
232           if (region.getEndKey().length > 0) {
233             end = Bytes.toInt(region.getEndKey());
234           }
235           int mid = start + ((end - start) / 2);
236           byte[] splitPoint = Bytes.toBytes(mid);
237 
238           //put some rows to the regions
239           addData(start);
240           addData(mid);
241 
242           flushAndBlockUntilDone(admin, rs, region.getRegionName());
243           compactAndBlockUntilDone(admin, rs, region.getRegionName());
244 
245           log("Initiating region split for:" + region.getRegionNameAsString());
246           try {
247             admin.split(region.getRegionName(), splitPoint);
248             //wait until the split is complete
249             blockUntilRegionSplit(conf, 50000, region.getRegionName(), true);
250 
251           } catch (NotServingRegionException ex) {
252             //ignore
253           }
254         }
255       } catch (Throwable ex) {
256         this.ex = ex;
257       }
258     }
259 
260     void addData(int start) throws IOException {
261       for (int i=start; i< start + 100; i++) {
262         Put put = new Put(Bytes.toBytes(i));
263 
264         put.add(family, family, Bytes.toBytes(i));
265         table.put(put);
266       }
267       table.flushCommits();
268     }
269   }
270 
271   /**
272    * Checks regions using MetaScanner, MetaReader and HTable methods
273    */
274   static class RegionChecker extends Chore {
275     Configuration conf;
276     byte[] tableName;
277     Throwable ex;
278 
279     RegionChecker(Configuration conf, Stoppable stopper, byte[] tableName) {
280       super("RegionChecker", 10, stopper);
281       this.conf = conf;
282       this.tableName = tableName;
283       this.setDaemon(true);
284     }
285 
286     /** verify region boundaries obtained from MetaScanner */
287     void verifyRegionsUsingMetaScanner() throws Exception {
288 
289       //MetaScanner.allTableRegions()
290       NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, null,
291           tableName, false);
292       verifyTableRegions(regions.keySet());
293 
294       //MetaScanner.listAllRegions()
295       List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, false);
296       verifyTableRegions(Sets.newTreeSet(regionList));
297     }
298 
299     /** verify region boundaries obtained from HTable.getStartEndKeys() */
300     void verifyRegionsUsingHTable() throws IOException {
301       HTable table = null;
302       try {
303         //HTable.getStartEndKeys()
304         table = new HTable(conf, tableName);
305         Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
306         verifyStartEndKeys(keys);
307 
308         //HTable.getRegionsInfo()
309         Map<HRegionInfo, HServerAddress> regions = table.getRegionsInfo();
310         verifyTableRegions(regions.keySet());
311       } finally {
312         IOUtils.closeQuietly(table);
313       }
314     }
315 
316     void verify() throws Exception {
317       verifyRegionsUsingMetaScanner();
318       verifyRegionsUsingHTable();
319     }
320 
321     void verifyTableRegions(Set<HRegionInfo> regions) {
322       log("Verifying " + regions.size() + " regions");
323 
324       byte[][] startKeys = new byte[regions.size()][];
325       byte[][] endKeys = new byte[regions.size()][];
326 
327       int i=0;
328       for (HRegionInfo region : regions) {
329         startKeys[i] = region.getStartKey();
330         endKeys[i] = region.getEndKey();
331         i++;
332       }
333 
334       Pair<byte[][], byte[][]> keys = new Pair<byte[][], byte[][]>(startKeys, endKeys);
335       verifyStartEndKeys(keys);
336     }
337 
338     void verifyStartEndKeys(Pair<byte[][], byte[][]> keys) {
339       byte[][] startKeys = keys.getFirst();
340       byte[][] endKeys = keys.getSecond();
341       assertEquals(startKeys.length, endKeys.length);
342       assertTrue("Found 0 regions for the table", startKeys.length > 0);
343 
344       assertArrayEquals("Start key for the first region is not byte[0]",
345           HConstants.EMPTY_START_ROW, startKeys[0]);
346       byte[] prevEndKey = HConstants.EMPTY_START_ROW;
347 
348       // ensure that we do not have any gaps
349       for (int i=0; i<startKeys.length; i++) {
350         assertArrayEquals(
351             "Hole in .META. is detected. prevEndKey=" + Bytes.toStringBinary(prevEndKey)
352                 + " ,regionStartKey=" + Bytes.toStringBinary(startKeys[i]), prevEndKey,
353             startKeys[i]);
354         prevEndKey = endKeys[i];
355       }
356       assertArrayEquals("End key for the last region is not byte[0]", HConstants.EMPTY_END_ROW,
357           endKeys[endKeys.length - 1]);
358     }
359 
360     @Override
361     protected void chore() {
362       try {
363         verify();
364       } catch (Throwable ex) {
365         this.ex = ex;
366         stopper.stop("caught exception");
367       }
368     }
369   }
370 
371   public static void log(String msg) {
372     LOG.info(msg);
373   }
374 
375   /* some utility methods for split tests */
376 
377   public static void flushAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
378       throws IOException, InterruptedException {
379     log("flushing region: " + Bytes.toStringBinary(regionName));
380     admin.flush(regionName);
381     log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
382     Threads.sleepWithoutInterrupt(500);
383     while (rs.cacheFlusher.getFlushQueueSize() > 0) {
384       Threads.sleep(50);
385     }
386   }
387 
388   public static void compactAndBlockUntilDone(HBaseAdmin admin, HRegionServer rs, byte[] regionName)
389       throws IOException, InterruptedException {
390     log("Compacting region: " + Bytes.toStringBinary(regionName));
391     admin.majorCompact(regionName);
392     log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
393     Threads.sleepWithoutInterrupt(500);
394     while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
395       Threads.sleep(50);
396     }
397   }
398 
399   /** Blocks until the region split is complete in META and region server opens the daughters */
400   public static void blockUntilRegionSplit(Configuration conf, long timeout,
401       final byte[] regionName, boolean waitForDaughters)
402       throws IOException, InterruptedException {
403     long start = System.currentTimeMillis();
404     log("blocking until region is split:" +  Bytes.toStringBinary(regionName));
405     HRegionInfo daughterA = null, daughterB = null;
406     HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
407 
408     try {
409       while (System.currentTimeMillis() - start < timeout) {
410         Result result = getRegionRow(metaTable, regionName);
411         if (result == null) {
412           break;
413         }
414 
415         HRegionInfo region = MetaReader.parseCatalogResult(result).getFirst();
416         if(region.isSplitParent()) {
417           log("found parent region: " + region.toString());
418           PairOfSameType<HRegionInfo> pair = MetaReader.getDaughterRegions(result);
419           daughterA = pair.getFirst();
420           daughterB = pair.getSecond();
421           break;
422         }
423         Threads.sleep(100);
424       }
425 
426       //if we are here, this means the region split is complete or timed out
427       if (waitForDaughters) {
428         long rem = timeout - (System.currentTimeMillis() - start);
429         blockUntilRegionIsInMeta(metaTable, rem, daughterA);
430 
431         rem = timeout - (System.currentTimeMillis() - start);
432         blockUntilRegionIsInMeta(metaTable, rem, daughterB);
433 
434         rem = timeout - (System.currentTimeMillis() - start);
435         blockUntilRegionIsOpened(conf, rem, daughterA);
436 
437         rem = timeout - (System.currentTimeMillis() - start);
438         blockUntilRegionIsOpened(conf, rem, daughterB);
439       }
440     } finally {
441       IOUtils.closeQuietly(metaTable);
442     }
443   }
444 
445   public static Result getRegionRow(HTable metaTable, byte[] regionName) throws IOException {
446     Get get = new Get(regionName);
447     return metaTable.get(get);
448   }
449 
450   public static void blockUntilRegionIsInMeta(HTable metaTable, long timeout, HRegionInfo hri)
451       throws IOException, InterruptedException {
452     log("blocking until region is in META: " + hri.getRegionNameAsString());
453     long start = System.currentTimeMillis();
454     while (System.currentTimeMillis() - start < timeout) {
455       Result result = getRegionRow(metaTable, hri.getRegionName());
456       if (result != null) {
457         HRegionInfo info = MetaReader.parseCatalogResult(result).getFirst();
458         if (info != null && !info.isOffline()) {
459           log("found region in META: " + hri.getRegionNameAsString());
460           break;
461         }
462       }
463       Threads.sleep(10);
464     }
465   }
466 
467   public static void blockUntilRegionIsOpened(Configuration conf, long timeout, HRegionInfo hri)
468       throws IOException, InterruptedException {
469     log("blocking until region is opened for reading:" + hri.getRegionNameAsString());
470     long start = System.currentTimeMillis();
471     HTable table = new HTable(conf, hri.getTableName());
472 
473     try {
474       Get get = new Get(hri.getStartKey());
475       while (System.currentTimeMillis() - start < timeout) {
476         try {
477           table.get(get);
478           break;
479         } catch(IOException ex) {
480           //wait some more
481         }
482         Threads.sleep(10);
483       }
484     } finally {
485       IOUtils.closeQuietly(table);
486     }
487   }
488 
489   @org.junit.Rule
490   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
491     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
492 }
493