View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.HashMap;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.concurrent.CountDownLatch;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.testclassification.LargeTests;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.TableNotFoundException;
44  import org.apache.hadoop.hbase.client.Admin;
45  import org.apache.hadoop.hbase.client.Table;
46  import org.apache.hadoop.hbase.master.HMaster;
47  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
48  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
49  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.FSUtils;
52  import org.junit.After;
53  import org.junit.AfterClass;
54  import org.junit.Before;
55  import org.junit.BeforeClass;
56  import org.junit.Test;
57  import org.junit.experimental.categories.Category;
58  
59  /**
60   * Test creating/using/deleting snapshots from the client
61   * <p>
62   * This is an end-to-end test for the snapshot utility
63   *
64   * TODO This is essentially a clone of TestSnapshotFromClient.  This is worth refactoring this
65   * because there will be a few more flavors of snapshots that need to run these tests.
66   */
67  @Category(LargeTests.class)
68  public class TestFlushSnapshotFromClient {
69    private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
70    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
71    private static final int NUM_RS = 2;
72    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
73    private static final TableName TABLE_NAME = TableName.valueOf("test");
74    private final int DEFAULT_NUM_ROWS = 100;
75  
76    /**
77     * Setup the config for the cluster
78     * @throws Exception on failure
79     */
80    @BeforeClass
81    public static void setupCluster() throws Exception {
82      // Uncomment the following lines if more verbosity is needed for
83      // debugging (see HBASE-12285 for details).
84      //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
85      //((Log4JLogger)AbstractRpcClient.LOG).getLogger().setLevel(Level.ALL);
86      //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
87      setupConf(UTIL.getConfiguration());
88      UTIL.startMiniCluster(NUM_RS);
89    }
90  
91    private static void setupConf(Configuration conf) {
92      // disable the ui
93      conf.setInt("hbase.regionsever.info.port", -1);
94      // change the flush size to a small amount, regulating number of store files
95      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
96      // so make sure we get a compaction when doing a load, but keep around some
97      // files in the store
98      conf.setInt("hbase.hstore.compaction.min", 10);
99      conf.setInt("hbase.hstore.compactionThreshold", 10);
100     // block writes if we get to 12 store files
101     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
102     // Enable snapshot
103     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
104     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
105       ConstantSizeRegionSplitPolicy.class.getName());
106   }
107 
108   @Before
109   public void setup() throws Exception {
110     SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
111   }
112 
113   @After
114   public void tearDown() throws Exception {
115     UTIL.deleteTable(TABLE_NAME);
116 
117     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
118     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
119   }
120 
121   @AfterClass
122   public static void cleanupTest() throws Exception {
123     try {
124       UTIL.shutdownMiniCluster();
125     } catch (Exception e) {
126       LOG.warn("failure shutting down cluster", e);
127     }
128   }
129 
130   /**
131    * Test simple flush snapshotting a table that is online
132    * @throws Exception
133    */
134   @Test (timeout=300000)
135   public void testFlushTableSnapshot() throws Exception {
136     Admin admin = UTIL.getHBaseAdmin();
137     // make sure we don't fail on listing snapshots
138     SnapshotTestingUtils.assertNoSnapshots(admin);
139 
140     // put some stuff in the table
141     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
142 
143     LOG.debug("FS state before snapshot:");
144     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
145         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
146 
147     // take a snapshot of the enabled table
148     String snapshotString = "offlineTableSnapshot";
149     byte[] snapshot = Bytes.toBytes(snapshotString);
150     admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH);
151     LOG.debug("Snapshot completed.");
152 
153     // make sure we have the snapshot
154     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
155       snapshot, TABLE_NAME);
156 
157     // make sure its a valid snapshot
158     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
159     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
160     LOG.debug("FS state after snapshot:");
161     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
162         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
163 
164     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
165         admin, fs);
166   }
167 
168    /**
169    * Test snapshotting a table that is online without flushing
170    * @throws Exception
171    */
172   @Test(timeout=30000)
173   public void testSkipFlushTableSnapshot() throws Exception {
174     Admin admin = UTIL.getHBaseAdmin();
175     // make sure we don't fail on listing snapshots
176     SnapshotTestingUtils.assertNoSnapshots(admin);
177 
178     // put some stuff in the table
179     try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
180       UTIL.loadTable(table, TEST_FAM);
181     }
182 
183     LOG.debug("FS state before snapshot:");
184     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
185         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
186 
187     // take a snapshot of the enabled table
188     String snapshotString = "skipFlushTableSnapshot";
189     byte[] snapshot = Bytes.toBytes(snapshotString);
190     admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
191     LOG.debug("Snapshot completed.");
192 
193     // make sure we have the snapshot
194     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
195         snapshot, TABLE_NAME);
196 
197     // make sure its a valid snapshot
198     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
199     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
200     LOG.debug("FS state after snapshot:");
201     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
202         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
203 
204     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
205         admin, fs);
206 
207     admin.deleteSnapshot(snapshot);
208     snapshots = admin.listSnapshots();
209     SnapshotTestingUtils.assertNoSnapshots(admin);
210   }
211 
212 
213   /**
214    * Test simple flush snapshotting a table that is online
215    * @throws Exception
216    */
217   @Test (timeout=300000)
218   public void testFlushTableSnapshotWithProcedure() throws Exception {
219     Admin admin = UTIL.getHBaseAdmin();
220     // make sure we don't fail on listing snapshots
221     SnapshotTestingUtils.assertNoSnapshots(admin);
222 
223     // put some stuff in the table
224     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
225 
226     LOG.debug("FS state before snapshot:");
227     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
228         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
229 
230     // take a snapshot of the enabled table
231     String snapshotString = "offlineTableSnapshot";
232     byte[] snapshot = Bytes.toBytes(snapshotString);
233     Map<String, String> props = new HashMap<String, String>();
234     props.put("table", TABLE_NAME.getNameAsString());
235     admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
236         snapshotString, props);
237 
238 
239     LOG.debug("Snapshot completed.");
240 
241     // make sure we have the snapshot
242     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
243       snapshot, TABLE_NAME);
244 
245     // make sure its a valid snapshot
246     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
247     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
248     LOG.debug("FS state after snapshot:");
249     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
250         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
251 
252     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
253         admin, fs);
254   }
255 
256   @Test (timeout=300000)
257   public void testSnapshotFailsOnNonExistantTable() throws Exception {
258     Admin admin = UTIL.getHBaseAdmin();
259     // make sure we don't fail on listing snapshots
260     SnapshotTestingUtils.assertNoSnapshots(admin);
261     TableName tableName = TableName.valueOf("_not_a_table");
262 
263     // make sure the table doesn't exist
264     boolean fail = false;
265     do {
266     try {
267       admin.getTableDescriptor(tableName);
268       fail = true;
269       LOG.error("Table:" + tableName + " already exists, checking a new name");
270       tableName = TableName.valueOf(tableName+"!");
271     } catch (TableNotFoundException e) {
272       fail = false;
273       }
274     } while (fail);
275 
276     // snapshot the non-existant table
277     try {
278       admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
279       fail("Snapshot succeeded even though there is not table.");
280     } catch (SnapshotCreationException e) {
281       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
282     }
283   }
284 
285   @Test(timeout = 300000)
286   public void testAsyncFlushSnapshot() throws Exception {
287     Admin admin = UTIL.getHBaseAdmin();
288     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
289         .setTable(TABLE_NAME.getNameAsString())
290         .setType(SnapshotDescription.Type.FLUSH)
291         .build();
292 
293     // take the snapshot async
294     admin.takeSnapshotAsync(snapshot);
295 
296     // constantly loop, looking for the snapshot to complete
297     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
298     SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
299     LOG.info(" === Async Snapshot Completed ===");
300     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
301       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
302     // make sure we get the snapshot
303     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
304   }
305 
306   @Test (timeout=300000)
307   public void testSnapshotStateAfterMerge() throws Exception {
308     int numRows = DEFAULT_NUM_ROWS;
309     Admin admin = UTIL.getHBaseAdmin();
310     // make sure we don't fail on listing snapshots
311     SnapshotTestingUtils.assertNoSnapshots(admin);
312     // load the table so we have some data
313     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
314 
315     // Take a snapshot
316     String snapshotBeforeMergeName = "snapshotBeforeMerge";
317     admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH);
318 
319     // Clone the table
320     TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge");
321     admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
322     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName);
323 
324     // Merge two regions
325     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
326     Collections.sort(regions, new Comparator<HRegionInfo>() {
327       public int compare(HRegionInfo r1, HRegionInfo r2) {
328         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
329       }
330     });
331 
332     int numRegions = admin.getTableRegions(TABLE_NAME).size();
333     int numRegionsAfterMerge = numRegions - 2;
334     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
335         regions.get(2).getEncodedNameAsBytes(), true);
336     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
337         regions.get(6).getEncodedNameAsBytes(), true);
338 
339     // Verify that there's one region less
340     waitRegionsAfterMerge(numRegionsAfterMerge);
341     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
342 
343     // Clone the table
344     TableName cloneAfterMergeName = TableName.valueOf("cloneAfterMerge");
345     admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
346     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName);
347 
348     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
349     SnapshotTestingUtils.verifyRowCount(UTIL, cloneBeforeMergeName, numRows);
350     SnapshotTestingUtils.verifyRowCount(UTIL, cloneAfterMergeName, numRows);
351 
352     // test that we can delete the snapshot
353     UTIL.deleteTable(cloneAfterMergeName);
354     UTIL.deleteTable(cloneBeforeMergeName);
355   }
356 
357   @Test (timeout=300000)
358   public void testTakeSnapshotAfterMerge() throws Exception {
359     int numRows = DEFAULT_NUM_ROWS;
360     Admin admin = UTIL.getHBaseAdmin();
361     // make sure we don't fail on listing snapshots
362     SnapshotTestingUtils.assertNoSnapshots(admin);
363     // load the table so we have some data
364     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
365 
366     // Merge two regions
367     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
368     Collections.sort(regions, new Comparator<HRegionInfo>() {
369       public int compare(HRegionInfo r1, HRegionInfo r2) {
370         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
371       }
372     });
373 
374     int numRegions = admin.getTableRegions(TABLE_NAME).size();
375     int numRegionsAfterMerge = numRegions - 2;
376     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
377         regions.get(2).getEncodedNameAsBytes(), true);
378     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
379         regions.get(6).getEncodedNameAsBytes(), true);
380 
381     waitRegionsAfterMerge(numRegionsAfterMerge);
382     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
383 
384     // Take a snapshot
385     String snapshotName = "snapshotAfterMerge";
386     SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(),
387       SnapshotDescription.Type.FLUSH, 3);
388 
389     // Clone the table
390     TableName cloneName = TableName.valueOf("cloneMerge");
391     admin.cloneSnapshot(snapshotName, cloneName);
392     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);
393 
394     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
395     SnapshotTestingUtils.verifyRowCount(UTIL, cloneName, numRows);
396 
397     // test that we can delete the snapshot
398     UTIL.deleteTable(cloneName);
399   }
400 
401   /**
402    * Basic end-to-end test of simple-flush-based snapshots
403    */
404   @Test (timeout=300000)
405   public void testFlushCreateListDestroy() throws Exception {
406     LOG.debug("------- Starting Snapshot test -------------");
407     Admin admin = UTIL.getHBaseAdmin();
408     // make sure we don't fail on listing snapshots
409     SnapshotTestingUtils.assertNoSnapshots(admin);
410     // load the table so we have some data
411     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
412 
413     String snapshotName = "flushSnapshotCreateListDestroy";
414     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
415     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
416     SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
417       snapshotName, rootDir, fs, true);
418   }
419 
420   /**
421    * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
422    * same table currently running and that concurrent snapshots on different tables can both
423    * succeed concurretly.
424    */
425   @Test(timeout=300000)
426   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
427     final TableName TABLE2_NAME = TableName.valueOf(TABLE_NAME + "2");
428 
429     int ssNum = 20;
430     Admin admin = UTIL.getHBaseAdmin();
431     // make sure we don't fail on listing snapshots
432     SnapshotTestingUtils.assertNoSnapshots(admin);
433     // create second testing table
434     SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
435     // load the table so we have some data
436     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
437     SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
438 
439     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
440     // We'll have one of these per thread
441     class SSRunnable implements Runnable {
442       SnapshotDescription ss;
443       SSRunnable(SnapshotDescription ss) {
444         this.ss = ss;
445       }
446 
447       @Override
448       public void run() {
449         try {
450           Admin admin = UTIL.getHBaseAdmin();
451           LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
452           admin.takeSnapshotAsync(ss);
453         } catch (Exception e) {
454           LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
455               ss)
456               + ".  This is ok, we expect some", e);
457         }
458         LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
459         toBeSubmitted.countDown();
460       }
461     };
462 
463     // build descriptions
464     SnapshotDescription[] descs = new SnapshotDescription[ssNum];
465     for (int i = 0; i < ssNum; i++) {
466       SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
467       builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
468       builder.setName("ss"+i);
469       builder.setType(SnapshotDescription.Type.FLUSH);
470       descs[i] = builder.build();
471     }
472 
473     // kick each off its own thread
474     for (int i=0 ; i < ssNum; i++) {
475       new Thread(new SSRunnable(descs[i])).start();
476     }
477 
478     // wait until all have been submitted
479     toBeSubmitted.await();
480 
481     // loop until all are done.
482     while (true) {
483       int doneCount = 0;
484       for (SnapshotDescription ss : descs) {
485         try {
486           if (admin.isSnapshotFinished(ss)) {
487             doneCount++;
488           }
489         } catch (Exception e) {
490           LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
491           doneCount++;
492         }
493       }
494       if (doneCount == descs.length) {
495         break;
496       }
497       Thread.sleep(100);
498     }
499 
500     // dump for debugging
501     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
502 
503     List<SnapshotDescription> taken = admin.listSnapshots();
504     int takenSize = taken.size();
505     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
506     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
507         " issued many requests", takenSize < ssNum && takenSize > 0);
508 
509     // Verify that there's at least one snapshot per table
510     int t1SnapshotsCount = 0;
511     int t2SnapshotsCount = 0;
512     for (SnapshotDescription ss : taken) {
513       if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
514         t1SnapshotsCount++;
515       } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
516         t2SnapshotsCount++;
517       }
518     }
519     assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
520     assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
521 
522     UTIL.deleteTable(TABLE2_NAME);
523   }
524 
525   private void logFSTree(Path root) throws IOException {
526     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
527   }
528 
529   private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
530       throws IOException, InterruptedException {
531     Admin admin = UTIL.getHBaseAdmin();
532     // Verify that there's one region less
533     long startTime = System.currentTimeMillis();
534     while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
535       // This may be flaky... if after 15sec the merge is not complete give up
536       // it will fail in the assertEquals(numRegionsAfterMerge).
537       if ((System.currentTimeMillis() - startTime) > 15000)
538         break;
539       Thread.sleep(100);
540     }
541     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
542   }
543 }