View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertArrayEquals;
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.NavigableMap;
30  import java.util.Random;
31  import java.util.Set;
32  
33  import org.apache.commons.io.IOUtils;
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.Chore;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.testclassification.LargeTests;
43  import org.apache.hadoop.hbase.NotServingRegionException;
44  import org.apache.hadoop.hbase.client.Admin;
45  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
46  import org.apache.hadoop.hbase.ServerName;
47  import org.apache.hadoop.hbase.Stoppable;
48  import org.apache.hadoop.hbase.TableName;
49  import org.apache.hadoop.hbase.client.Admin;
50  import org.apache.hadoop.hbase.client.Connection;
51  import org.apache.hadoop.hbase.client.ConnectionFactory;
52  import org.apache.hadoop.hbase.client.Get;
53  import org.apache.hadoop.hbase.client.HConnection;
54  import org.apache.hadoop.hbase.client.HConnectionManager;
55  import org.apache.hadoop.hbase.client.HTable;
56  import org.apache.hadoop.hbase.client.MetaScanner;
57  import org.apache.hadoop.hbase.client.Put;
58  import org.apache.hadoop.hbase.client.Result;
59  import org.apache.hadoop.hbase.client.Scan;
60  import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
61  import org.apache.hadoop.hbase.client.Table;
62  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
63  import org.apache.hadoop.hbase.protobuf.RequestConverter;
64  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
65  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
66  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
67  import org.apache.hadoop.hbase.util.Bytes;
68  import org.apache.hadoop.hbase.util.ConfigUtil;
69  import org.apache.hadoop.hbase.util.Pair;
70  import org.apache.hadoop.hbase.util.PairOfSameType;
71  import org.apache.hadoop.hbase.util.StoppableImplementation;
72  import org.apache.hadoop.hbase.util.Threads;
73  import org.junit.AfterClass;
74  import org.junit.BeforeClass;
75  import org.junit.Test;
76  import org.junit.experimental.categories.Category;
77  
78  import com.google.common.collect.Iterators;
79  import com.google.common.collect.Sets;
80  import com.google.protobuf.ServiceException;
81  
82  @Category(LargeTests.class)
83  public class TestEndToEndSplitTransaction {
84    private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class);
85    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
86    private static final Configuration conf = TEST_UTIL.getConfiguration();
87  
88    @BeforeClass
89    public static void beforeAllTests() throws Exception {
90      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
91      TEST_UTIL.startMiniCluster();
92    }
93  
94    @AfterClass
95    public static void afterAllTests() throws Exception {
96      TEST_UTIL.shutdownMiniCluster();
97    }
98  
99    @Test
100   public void testMasterOpsWhileSplitting() throws Exception {
101     TableName tableName =
102         TableName.valueOf("TestSplit");
103     byte[] familyName = Bytes.toBytes("fam");
104     try (HTable ht = TEST_UTIL.createTable(tableName, familyName)) {
105       TEST_UTIL.loadTable(ht, familyName, false);
106     }
107     HRegionServer server = TEST_UTIL.getHBaseCluster().getRegionServer(0);
108     byte []firstRow = Bytes.toBytes("aaa");
109     byte []splitRow = Bytes.toBytes("lll");
110     byte []lastRow = Bytes.toBytes("zzz");
111     HConnection con = HConnectionManager
112         .getConnection(TEST_UTIL.getConfiguration());
113     // this will also cache the region
114     byte[] regionName = con.locateRegion(tableName, splitRow).getRegionInfo()
115         .getRegionName();
116     HRegion region = server.getRegion(regionName);
117     SplitTransaction split = new SplitTransaction(region, splitRow);
118     split.useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
119     split.prepare();
120 
121     // 1. phase I
122     PairOfSameType<HRegion> regions = split.createDaughters(server, server);
123     assertFalse(test(con, tableName, firstRow, server));
124     assertFalse(test(con, tableName, lastRow, server));
125 
126     // passing null as services prevents final step
127     // 2, most of phase II
128     split.openDaughters(server, null, regions.getFirst(), regions.getSecond());
129     assertFalse(test(con, tableName, firstRow, server));
130     assertFalse(test(con, tableName, lastRow, server));
131 
132     // 3. finish phase II
133     // note that this replicates some code from SplitTransaction
134     // 2nd daughter first
135     if (split.useZKForAssignment) {
136       server.postOpenDeployTasks(regions.getSecond());
137     } else {
138     server.reportRegionStateTransition(
139       RegionServerStatusProtos.RegionStateTransition.TransitionCode.SPLIT,
140       region.getRegionInfo(), regions.getFirst().getRegionInfo(),
141       regions.getSecond().getRegionInfo());
142     }
143 
144     // Add to online regions
145     server.addToOnlineRegions(regions.getSecond());
146     // THIS is the crucial point:
147     // the 2nd daughter was added, so querying before the split key should fail.
148     assertFalse(test(con, tableName, firstRow, server));
149     // past splitkey is ok.
150     assertTrue(test(con, tableName, lastRow, server));
151 
152     // first daughter second
153     if (split.useZKForAssignment) {
154       server.postOpenDeployTasks(regions.getFirst());
155     }
156     // Add to online regions
157     server.addToOnlineRegions(regions.getFirst());
158     assertTrue(test(con, tableName, firstRow, server));
159     assertTrue(test(con, tableName, lastRow, server));
160 
161     if (split.useZKForAssignment) {
162       // 4. phase III
163       ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
164         .getSplitTransactionCoordination().completeSplitTransaction(server, regions.getFirst(),
165           regions.getSecond(), split.std, region);
166     }
167     assertTrue(test(con, tableName, firstRow, server));
168     assertTrue(test(con, tableName, lastRow, server));
169   }
170 
171   /**
172    * attempt to locate the region and perform a get and scan
173    * @return True if successful, False otherwise.
174    */
175   private boolean test(HConnection con, TableName tableName, byte[] row,
176       HRegionServer server) {
177     // not using HTable to avoid timeouts and retries
178     try {
179       byte[] regionName = con.relocateRegion(tableName, row).getRegionInfo()
180           .getRegionName();
181       // get and scan should now succeed without exception
182       ClientProtos.GetRequest request =
183           RequestConverter.buildGetRequest(regionName, new Get(row));
184       server.getRSRpcServices().get(null, request);
185       ScanRequest scanRequest = RequestConverter.buildScanRequest(
186         regionName, new Scan(row), 1, true);
187       try {
188         server.getRSRpcServices().scan(
189           new PayloadCarryingRpcController(), scanRequest);
190       } catch (ServiceException se) {
191         throw ProtobufUtil.getRemoteException(se);
192       }
193     } catch (IOException x) {
194       return false;
195     } catch (ServiceException e) {
196       return false;
197     }
198     return true;
199   }
200 
201   /**
202    * Tests that the client sees meta table changes as atomic during splits
203    */
204   @Test
205   public void testFromClientSideWhileSplitting() throws Throwable {
206     LOG.info("Starting testFromClientSideWhileSplitting");
207     final TableName TABLENAME =
208         TableName.valueOf("testFromClientSideWhileSplitting");
209     final byte[] FAMILY = Bytes.toBytes("family");
210 
211     //SplitTransaction will update the meta table by offlining the parent region, and adding info
212     //for daughters.
213     Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
214 
215     Stoppable stopper = new StoppableImplementation();
216     RegionSplitter regionSplitter = new RegionSplitter(table);
217     RegionChecker regionChecker = new RegionChecker(conf, stopper, TABLENAME);
218 
219     regionChecker.start();
220     regionSplitter.start();
221 
222     //wait until the splitter is finished
223     regionSplitter.join();
224     stopper.stop(null);
225 
226     if (regionChecker.ex != null) {
227       throw regionChecker.ex;
228     }
229 
230     if (regionSplitter.ex != null) {
231       throw regionSplitter.ex;
232     }
233 
234     //one final check
235     regionChecker.verify();
236   }
237 
238   static class RegionSplitter extends Thread {
239     final Connection connection;
240     Throwable ex;
241     Table table;
242     TableName tableName;
243     byte[] family;
244     Admin admin;
245     HRegionServer rs;
246 
247     RegionSplitter(Table table) throws IOException {
248       this.table = table;
249       this.tableName = table.getName();
250       this.family = table.getTableDescriptor().getFamiliesKeys().iterator().next();
251       admin = TEST_UTIL.getHBaseAdmin();
252       rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
253       connection = TEST_UTIL.getConnection();
254     }
255 
256     @Override
257     public void run() {
258       try {
259         Random random = new Random();
260         for (int i= 0; i< 5; i++) {
261           NavigableMap<HRegionInfo, ServerName> regions =
262               MetaScanner.allTableRegions(connection, tableName);
263           if (regions.size() == 0) {
264             continue;
265           }
266           int regionIndex = random.nextInt(regions.size());
267 
268           //pick a random region and split it into two
269           HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex);
270 
271           //pick the mid split point
272           int start = 0, end = Integer.MAX_VALUE;
273           if (region.getStartKey().length > 0) {
274             start = Bytes.toInt(region.getStartKey());
275           }
276           if (region.getEndKey().length > 0) {
277             end = Bytes.toInt(region.getEndKey());
278           }
279           int mid = start + ((end - start) / 2);
280           byte[] splitPoint = Bytes.toBytes(mid);
281 
282           //put some rows to the regions
283           addData(start);
284           addData(mid);
285 
286           flushAndBlockUntilDone(admin, rs, region.getRegionName());
287           compactAndBlockUntilDone(admin, rs, region.getRegionName());
288 
289           log("Initiating region split for:" + region.getRegionNameAsString());
290           try {
291             admin.splitRegion(region.getRegionName(), splitPoint);
292             //wait until the split is complete
293             blockUntilRegionSplit(conf, 50000, region.getRegionName(), true);
294 
295           } catch (NotServingRegionException ex) {
296             //ignore
297           }
298         }
299       } catch (Throwable ex) {
300         this.ex = ex;
301       }
302     }
303 
304     void addData(int start) throws IOException {
305       List<Put> puts = new ArrayList<>();
306       for (int i=start; i< start + 100; i++) {
307         Put put = new Put(Bytes.toBytes(i));
308         put.add(family, family, Bytes.toBytes(i));
309         puts.add(put);
310       }
311       table.put(puts);
312     }
313   }
314 
315   /**
316    * Checks regions using MetaScanner, MetaTableAccessor and HTable methods
317    */
318   static class RegionChecker extends Chore {
319     Connection connection;
320     Configuration conf;
321     TableName tableName;
322     Throwable ex;
323 
324     RegionChecker(Configuration conf, Stoppable stopper, TableName tableName) throws IOException {
325       super("RegionChecker", 10, stopper);
326       this.conf = conf;
327       this.tableName = tableName;
328       this.setDaemon(true);
329 
330       this.connection = ConnectionFactory.createConnection(conf);
331     }
332 
333     /** verify region boundaries obtained from MetaScanner */
334     void verifyRegionsUsingMetaScanner() throws Exception {
335 
336       //MetaScanner.allTableRegions()
337       NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(connection,
338           tableName);
339       verifyTableRegions(regions.keySet());
340 
341       //MetaScanner.listAllRegions()
342       List<HRegionInfo> regionList = MetaScanner.listAllRegions(conf, connection, false);
343       verifyTableRegions(Sets.newTreeSet(regionList));
344     }
345 
346     /** verify region boundaries obtained from HTable.getStartEndKeys() */
347     void verifyRegionsUsingHTable() throws IOException {
348       HTable table = null;
349       try {
350         //HTable.getStartEndKeys()
351         table = new HTable(conf, tableName);
352         Pair<byte[][], byte[][]> keys = table.getStartEndKeys();
353         verifyStartEndKeys(keys);
354 
355         //HTable.getRegionsInfo()
356         Map<HRegionInfo, ServerName> regions = table.getRegionLocations();
357         verifyTableRegions(regions.keySet());
358       } finally {
359         IOUtils.closeQuietly(table);
360       }
361     }
362 
363     void verify() throws Exception {
364       verifyRegionsUsingMetaScanner();
365       verifyRegionsUsingHTable();
366     }
367 
368     void verifyTableRegions(Set<HRegionInfo> regions) {
369       log("Verifying " + regions.size() + " regions: " + regions);
370 
371       byte[][] startKeys = new byte[regions.size()][];
372       byte[][] endKeys = new byte[regions.size()][];
373 
374       int i=0;
375       for (HRegionInfo region : regions) {
376         startKeys[i] = region.getStartKey();
377         endKeys[i] = region.getEndKey();
378         i++;
379       }
380 
381       Pair<byte[][], byte[][]> keys = new Pair<byte[][], byte[][]>(startKeys, endKeys);
382       verifyStartEndKeys(keys);
383     }
384 
385     void verifyStartEndKeys(Pair<byte[][], byte[][]> keys) {
386       byte[][] startKeys = keys.getFirst();
387       byte[][] endKeys = keys.getSecond();
388       assertEquals(startKeys.length, endKeys.length);
389       assertTrue("Found 0 regions for the table", startKeys.length > 0);
390 
391       assertArrayEquals("Start key for the first region is not byte[0]",
392           HConstants.EMPTY_START_ROW, startKeys[0]);
393       byte[] prevEndKey = HConstants.EMPTY_START_ROW;
394 
395       // ensure that we do not have any gaps
396       for (int i=0; i<startKeys.length; i++) {
397         assertArrayEquals(
398             "Hole in hbase:meta is detected. prevEndKey=" + Bytes.toStringBinary(prevEndKey)
399                 + " ,regionStartKey=" + Bytes.toStringBinary(startKeys[i]), prevEndKey,
400             startKeys[i]);
401         prevEndKey = endKeys[i];
402       }
403       assertArrayEquals("End key for the last region is not byte[0]", HConstants.EMPTY_END_ROW,
404           endKeys[endKeys.length - 1]);
405     }
406 
407     @Override
408     protected void chore() {
409       try {
410         verify();
411       } catch (Throwable ex) {
412         this.ex = ex;
413         stopper.stop("caught exception");
414       }
415     }
416   }
417 
418   public static void log(String msg) {
419     LOG.info(msg);
420   }
421 
422   /* some utility methods for split tests */
423 
424   public static void flushAndBlockUntilDone(Admin admin, HRegionServer rs, byte[] regionName)
425       throws IOException, InterruptedException {
426     log("flushing region: " + Bytes.toStringBinary(regionName));
427     admin.flushRegion(regionName);
428     log("blocking until flush is complete: " + Bytes.toStringBinary(regionName));
429     Threads.sleepWithoutInterrupt(500);
430     while (rs.cacheFlusher.getFlushQueueSize() > 0) {
431       Threads.sleep(50);
432     }
433   }
434 
435   public static void compactAndBlockUntilDone(Admin admin, HRegionServer rs, byte[] regionName)
436       throws IOException, InterruptedException {
437     log("Compacting region: " + Bytes.toStringBinary(regionName));
438     admin.majorCompactRegion(regionName);
439     log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName));
440     Threads.sleepWithoutInterrupt(500);
441     while (rs.compactSplitThread.getCompactionQueueSize() > 0) {
442       Threads.sleep(50);
443     }
444   }
445 
446   /** Blocks until the region split is complete in hbase:meta and region server opens the daughters */
447   public static void blockUntilRegionSplit(Configuration conf, long timeout,
448       final byte[] regionName, boolean waitForDaughters)
449       throws IOException, InterruptedException {
450     long start = System.currentTimeMillis();
451     log("blocking until region is split:" +  Bytes.toStringBinary(regionName));
452     HRegionInfo daughterA = null, daughterB = null;
453     Table metaTable = new HTable(conf, TableName.META_TABLE_NAME);
454 
455     try {
456       Result result = null;
457       HRegionInfo region = null;
458       while ((System.currentTimeMillis() - start) < timeout) {
459         result = getRegionRow(metaTable, regionName);
460         if (result == null) {
461           break;
462         }
463 
464         region = HRegionInfo.getHRegionInfo(result);
465         if (region.isSplitParent()) {
466           log("found parent region: " + region.toString());
467           PairOfSameType<HRegionInfo> pair = HRegionInfo.getDaughterRegions(result);
468           daughterA = pair.getFirst();
469           daughterB = pair.getSecond();
470           break;
471         }
472         Threads.sleep(100);
473       }
474       if (daughterA == null || daughterB == null) {
475         throw new IOException("Failed to get daughters, daughterA=" + daughterA + ", daughterB=" +
476           daughterB + ", timeout=" + timeout + ", result=" + result + ", regionName=" + regionName +
477           ", region=" + region);
478       }
479 
480       //if we are here, this means the region split is complete or timed out
481       if (waitForDaughters) {
482         long rem = timeout - (System.currentTimeMillis() - start);
483         blockUntilRegionIsInMeta(metaTable, rem, daughterA);
484 
485         rem = timeout - (System.currentTimeMillis() - start);
486         blockUntilRegionIsInMeta(metaTable, rem, daughterB);
487 
488         rem = timeout - (System.currentTimeMillis() - start);
489         blockUntilRegionIsOpened(conf, rem, daughterA);
490 
491         rem = timeout - (System.currentTimeMillis() - start);
492         blockUntilRegionIsOpened(conf, rem, daughterB);
493       }
494     } finally {
495       IOUtils.closeQuietly(metaTable);
496     }
497   }
498 
499   public static Result getRegionRow(Table metaTable, byte[] regionName) throws IOException {
500     Get get = new Get(regionName);
501     return metaTable.get(get);
502   }
503 
504   public static void blockUntilRegionIsInMeta(Table metaTable, long timeout, HRegionInfo hri)
505       throws IOException, InterruptedException {
506     log("blocking until region is in META: " + hri.getRegionNameAsString());
507     long start = System.currentTimeMillis();
508     while (System.currentTimeMillis() - start < timeout) {
509       Result result = getRegionRow(metaTable, hri.getRegionName());
510       if (result != null) {
511         HRegionInfo info = HRegionInfo.getHRegionInfo(result);
512         if (info != null && !info.isOffline()) {
513           log("found region in META: " + hri.getRegionNameAsString());
514           break;
515         }
516       }
517       Threads.sleep(10);
518     }
519   }
520 
521   public static void blockUntilRegionIsOpened(Configuration conf, long timeout, HRegionInfo hri)
522       throws IOException, InterruptedException {
523     log("blocking until region is opened for reading:" + hri.getRegionNameAsString());
524     long start = System.currentTimeMillis();
525     Table table = new HTable(conf, hri.getTable());
526 
527     try {
528       byte [] row = hri.getStartKey();
529       // Check for null/empty row.  If we find one, use a key that is likely to be in first region.
530       if (row == null || row.length <= 0) row = new byte [] {'0'};
531       Get get = new Get(row);
532       while (System.currentTimeMillis() - start < timeout) {
533         try {
534           table.get(get);
535           break;
536         } catch(IOException ex) {
537           //wait some more
538         }
539         Threads.sleep(10);
540       }
541     } finally {
542       IOUtils.closeQuietly(table);
543     }
544   }
545 }
546