View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.HashSet;
28  import java.util.List;
29  import java.util.Set;
30  import java.util.concurrent.CountDownLatch;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.commons.logging.impl.Log4JLogger;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.LargeTests;
44  import org.apache.hadoop.hbase.TableNotFoundException;
45  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
46  import org.apache.hadoop.hbase.ipc.RpcClient;
47  import org.apache.hadoop.hbase.ipc.RpcServer;
48  import org.apache.hadoop.hbase.client.HBaseAdmin;
49  import org.apache.hadoop.hbase.client.HTable;
50  import org.apache.hadoop.hbase.client.ScannerCallable;
51  import org.apache.hadoop.hbase.master.HMaster;
52  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
53  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
54  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
55  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.util.FSTableDescriptors;
58  import org.apache.hadoop.hbase.util.FSUtils;
59  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
60  import org.apache.log4j.Level;
61  import org.junit.After;
62  import org.junit.AfterClass;
63  import org.junit.Before;
64  import org.junit.BeforeClass;
65  import org.junit.Test;
66  import org.junit.experimental.categories.Category;
67  
68  /**
69   * Test creating/using/deleting snapshots from the client
70   * <p>
71   * This is an end-to-end test for the snapshot utility
72   *
73   * TODO This is essentially a clone of TestSnapshotFromClient.  This is worth refactoring this
74   * because there will be a few more flavors of snapshots that need to run these tests.
75   */
76  @Category(LargeTests.class)
77  public class TestFlushSnapshotFromClient {
78    private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
79    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
80    private static final int NUM_RS = 2;
81    private static final String STRING_TABLE_NAME = "test";
82    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
83    private static final byte[] TEST_QUAL = Bytes.toBytes("q");
84    private static final TableName TABLE_NAME =
85        TableName.valueOf(STRING_TABLE_NAME);
86    private final int DEFAULT_NUM_ROWS = 100;
87  
88    /**
89     * Setup the config for the cluster
90     * @throws Exception on failure
91     */
92    @BeforeClass
93    public static void setupCluster() throws Exception {
94      ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
95      ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
96      ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
97      setupConf(UTIL.getConfiguration());
98      UTIL.startMiniCluster(NUM_RS);
99    }
100 
101   private static void setupConf(Configuration conf) {
102     // disable the ui
103     conf.setInt("hbase.regionsever.info.port", -1);
104     // change the flush size to a small amount, regulating number of store files
105     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
106     // so make sure we get a compaction when doing a load, but keep around some
107     // files in the store
108     conf.setInt("hbase.hstore.compaction.min", 10);
109     conf.setInt("hbase.hstore.compactionThreshold", 10);
110     // block writes if we get to 12 store files
111     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
112     // Enable snapshot
113     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
114     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
115       ConstantSizeRegionSplitPolicy.class.getName());
116   }
117 
118   @Before
119   public void setup() throws Exception {
120     SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
121   }
122 
123   @After
124   public void tearDown() throws Exception {
125     UTIL.deleteTable(TABLE_NAME);
126 
127     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
128     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
129   }
130 
131   @AfterClass
132   public static void cleanupTest() throws Exception {
133     try {
134       UTIL.shutdownMiniCluster();
135     } catch (Exception e) {
136       LOG.warn("failure shutting down cluster", e);
137     }
138   }
139 
140   /**
141    * Test simple flush snapshotting a table that is online
142    * @throws Exception
143    */
144   @Test (timeout=300000)
145   public void testFlushTableSnapshot() throws Exception {
146     HBaseAdmin admin = UTIL.getHBaseAdmin();
147     // make sure we don't fail on listing snapshots
148     SnapshotTestingUtils.assertNoSnapshots(admin);
149 
150     // put some stuff in the table
151     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
152     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
153 
154     // get the name of all the regionservers hosting the snapshotted table
155     Set<String> snapshotServers = new HashSet<String>();
156     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
157     for (RegionServerThread server : servers) {
158       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
159         snapshotServers.add(server.getRegionServer().getServerName().toString());
160       }
161     }
162 
163     LOG.debug("FS state before snapshot:");
164     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
165         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
166 
167     // take a snapshot of the enabled table
168     String snapshotString = "offlineTableSnapshot";
169     byte[] snapshot = Bytes.toBytes(snapshotString);
170     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
171     LOG.debug("Snapshot completed.");
172 
173     // make sure we have the snapshot
174     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
175       snapshot, TABLE_NAME);
176 
177     // make sure its a valid snapshot
178     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
179     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
180     LOG.debug("FS state after snapshot:");
181     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
182       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
183 
184     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
185       admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
186 
187     admin.deleteSnapshot(snapshot);
188     snapshots = admin.listSnapshots();
189     SnapshotTestingUtils.assertNoSnapshots(admin);
190   }
191 
192   @Test (timeout=300000)
193   public void testSnapshotFailsOnNonExistantTable() throws Exception {
194     HBaseAdmin admin = UTIL.getHBaseAdmin();
195     // make sure we don't fail on listing snapshots
196     SnapshotTestingUtils.assertNoSnapshots(admin);
197     String tableName = "_not_a_table";
198 
199     // make sure the table doesn't exist
200     boolean fail = false;
201     do {
202     try {
203       admin.getTableDescriptor(Bytes.toBytes(tableName));
204       fail = true;
205       LOG.error("Table:" + tableName + " already exists, checking a new name");
206       tableName = tableName+"!";
207     } catch (TableNotFoundException e) {
208       fail = false;
209       }
210     } while (fail);
211 
212     // snapshot the non-existant table
213     try {
214       admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
215       fail("Snapshot succeeded even though there is not table.");
216     } catch (SnapshotCreationException e) {
217       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
218     }
219   }
220 
221   @Test(timeout = 300000)
222   public void testAsyncFlushSnapshot() throws Exception {
223     HBaseAdmin admin = UTIL.getHBaseAdmin();
224     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
225         .setTable(TABLE_NAME.getNameAsString())
226         .setType(SnapshotDescription.Type.FLUSH)
227         .build();
228 
229     // take the snapshot async
230     admin.takeSnapshotAsync(snapshot);
231 
232     // constantly loop, looking for the snapshot to complete
233     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
234     SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
235     LOG.info(" === Async Snapshot Completed ===");
236     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
237       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
238     // make sure we get the snapshot
239     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
240 
241     // test that we can delete the snapshot
242     admin.deleteSnapshot(snapshot.getName());
243     LOG.info(" === Async Snapshot Deleted ===");
244     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
245       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
246     // make sure we don't have any snapshots
247     SnapshotTestingUtils.assertNoSnapshots(admin);
248     LOG.info(" === Async Snapshot Test Completed ===");
249 
250   }
251 
252   @Test (timeout=300000)
253   public void testSnapshotStateAfterMerge() throws Exception {
254     int numRows = DEFAULT_NUM_ROWS;
255     HBaseAdmin admin = UTIL.getHBaseAdmin();
256     // make sure we don't fail on listing snapshots
257     SnapshotTestingUtils.assertNoSnapshots(admin);
258     // load the table so we have some data
259     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
260 
261     // Take a snapshot
262     String snapshotBeforeMergeName = "snapshotBeforeMerge";
263     admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
264 
265     // Clone the table
266     String cloneBeforeMergeName = "cloneBeforeMerge";
267     admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
268     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
269 
270     // Merge two regions
271     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
272     Collections.sort(regions, new Comparator<HRegionInfo>() {
273       public int compare(HRegionInfo r1, HRegionInfo r2) {
274         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
275       }
276     });
277 
278     int numRegions = admin.getTableRegions(TABLE_NAME).size();
279     int numRegionsAfterMerge = numRegions - 2;
280     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
281         regions.get(2).getEncodedNameAsBytes(), true);
282     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
283         regions.get(6).getEncodedNameAsBytes(), true);
284 
285     // Verify that there's one region less
286     waitRegionsAfterMerge(numRegionsAfterMerge);
287     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
288 
289     // Clone the table
290     String cloneAfterMergeName = "cloneAfterMerge";
291     admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
292     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
293 
294     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
295     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
296     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
297 
298     // test that we can delete the snapshot
299     UTIL.deleteTable(cloneAfterMergeName);
300     UTIL.deleteTable(cloneBeforeMergeName);
301     admin.deleteSnapshot(snapshotBeforeMergeName);
302 
303     // make sure we don't have any snapshots
304     SnapshotTestingUtils.assertNoSnapshots(admin);
305   }
306 
307   @Test (timeout=300000)
308   public void testTakeSnapshotAfterMerge() throws Exception {
309     int numRows = DEFAULT_NUM_ROWS;
310     HBaseAdmin admin = UTIL.getHBaseAdmin();
311     // make sure we don't fail on listing snapshots
312     SnapshotTestingUtils.assertNoSnapshots(admin);
313     // load the table so we have some data
314     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
315 
316     // Merge two regions
317     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
318     Collections.sort(regions, new Comparator<HRegionInfo>() {
319       public int compare(HRegionInfo r1, HRegionInfo r2) {
320         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
321       }
322     });
323 
324     int numRegions = admin.getTableRegions(TABLE_NAME).size();
325     int numRegionsAfterMerge = numRegions - 2;
326     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
327         regions.get(2).getEncodedNameAsBytes(), true);
328     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
329         regions.get(6).getEncodedNameAsBytes(), true);
330 
331     waitRegionsAfterMerge(numRegionsAfterMerge);
332     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
333 
334     // Take a snapshot
335     String snapshotName = "snapshotAfterMerge";
336     admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
337 
338     // Clone the table
339     String cloneName = "cloneMerge";
340     admin.cloneSnapshot(snapshotName, cloneName);
341     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
342 
343     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
344     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
345 
346     // test that we can delete the snapshot
347     UTIL.deleteTable(cloneName);
348     admin.deleteSnapshot(snapshotName);
349 
350     // make sure we don't have any snapshots
351     SnapshotTestingUtils.assertNoSnapshots(admin);
352   }
353 
354   /**
355    * Basic end-to-end test of simple-flush-based snapshots
356    */
357   @Test (timeout=300000)
358   public void testFlushCreateListDestroy() throws Exception {
359     LOG.debug("------- Starting Snapshot test -------------");
360     HBaseAdmin admin = UTIL.getHBaseAdmin();
361     // make sure we don't fail on listing snapshots
362     SnapshotTestingUtils.assertNoSnapshots(admin);
363     // load the table so we have some data
364     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
365 
366     String snapshotName = "flushSnapshotCreateListDestroy";
367     // test creating the snapshot
368     admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
369     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
370 
371     // make sure we only have 1 matching snapshot
372     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
373       snapshotName, TABLE_NAME);
374 
375     // check the directory structure
376     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
377     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
378     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
379     assertTrue(fs.exists(snapshotDir));
380     FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
381     Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
382     assertTrue(fs.exists(snapshotinfo));
383 
384     // check the table info
385     HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs,
386         rootDir, TABLE_NAME);
387     HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs,
388         new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName));
389     assertEquals(desc, snapshotDesc);
390 
391     // check the region snapshot for all the regions
392     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
393     assertTrue(regions.size() > 1);
394     for (HRegionInfo info : regions) {
395       String regionName = info.getEncodedName();
396       Path regionDir = new Path(snapshotDir, regionName);
397       HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
398       assertEquals(info, snapshotRegionInfo);
399       // check to make sure we have the family
400       Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
401       assertTrue("Missing region " + Bytes.toString(snapshotRegionInfo.getStartKey()),
402                  fs.exists(familyDir));
403 
404       // make sure we have some file references
405       assertTrue(fs.listStatus(familyDir).length > 0);
406     }
407 
408     // test that we can delete the snapshot
409     admin.deleteSnapshot(snapshotName);
410     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
411       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
412 
413     // make sure we don't have any snapshots
414     SnapshotTestingUtils.assertNoSnapshots(admin);
415     LOG.debug("------- Flush-Snapshot Create List Destroy-------------");
416   }
417 
418   /**
419    * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
420    * same table currently running and that concurrent snapshots on different tables can both
421    * succeed concurretly.
422    */
423   @Test(timeout=300000)
424   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
425     final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
426     final TableName TABLE2_NAME =
427         TableName.valueOf(STRING_TABLE2_NAME);
428 
429     int ssNum = 20;
430     HBaseAdmin admin = UTIL.getHBaseAdmin();
431     // make sure we don't fail on listing snapshots
432     SnapshotTestingUtils.assertNoSnapshots(admin);
433     // create second testing table
434     SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
435     // load the table so we have some data
436     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
437     SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
438 
439     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
440     // We'll have one of these per thread
441     class SSRunnable implements Runnable {
442       SnapshotDescription ss;
443       SSRunnable(SnapshotDescription ss) {
444         this.ss = ss;
445       }
446 
447       @Override
448       public void run() {
449         try {
450           HBaseAdmin admin = UTIL.getHBaseAdmin();
451           LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
452           admin.takeSnapshotAsync(ss);
453         } catch (Exception e) {
454           LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
455               ss)
456               + ".  This is ok, we expect some", e);
457         }
458         LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
459         toBeSubmitted.countDown();
460       }
461     };
462 
463     // build descriptions
464     SnapshotDescription[] descs = new SnapshotDescription[ssNum];
465     for (int i = 0; i < ssNum; i++) {
466       SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
467       builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
468       builder.setName("ss"+i);
469       builder.setType(SnapshotDescription.Type.FLUSH);
470       descs[i] = builder.build();
471     }
472 
473     // kick each off its own thread
474     for (int i=0 ; i < ssNum; i++) {
475       new Thread(new SSRunnable(descs[i])).start();
476     }
477 
478     // wait until all have been submitted
479     toBeSubmitted.await();
480 
481     // loop until all are done.
482     while (true) {
483       int doneCount = 0;
484       for (SnapshotDescription ss : descs) {
485         try {
486           if (admin.isSnapshotFinished(ss)) {
487             doneCount++;
488           }
489         } catch (Exception e) {
490           LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
491           doneCount++;
492         }
493       }
494       if (doneCount == descs.length) {
495         break;
496       }
497       Thread.sleep(100);
498     }
499 
500     // dump for debugging
501     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
502 
503     List<SnapshotDescription> taken = admin.listSnapshots();
504     int takenSize = taken.size();
505     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
506     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
507         " issued many requests", takenSize < ssNum && takenSize > 0);
508 
509     // Verify that there's at least one snapshot per table
510     int t1SnapshotsCount = 0;
511     int t2SnapshotsCount = 0;
512     for (SnapshotDescription ss : taken) {
513       if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
514         t1SnapshotsCount++;
515       } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
516         t2SnapshotsCount++;
517       }
518     }
519     assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
520     assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
521 
522     // delete snapshots so subsequent tests are clean.
523     for (SnapshotDescription ss : taken) {
524       admin.deleteSnapshot(ss.getName());
525     }
526     UTIL.deleteTable(TABLE2_NAME);
527   }
528 
529   private void logFSTree(Path root) throws IOException {
530     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
531   }
532 
533   private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
534       throws IOException, InterruptedException {
535     HBaseAdmin admin = UTIL.getHBaseAdmin();
536     // Verify that there's one region less
537     long startTime = System.currentTimeMillis();
538     while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
539       // This may be flaky... if after 15sec the merge is not complete give up
540       // it will fail in the assertEquals(numRegionsAfterMerge).
541       if ((System.currentTimeMillis() - startTime) > 15000)
542         break;
543       Thread.sleep(100);
544     }
545     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
546   }
547 }