View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.HashMap;
28  import java.util.HashSet;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Set;
32  import java.util.concurrent.CountDownLatch;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.commons.logging.impl.Log4JLogger;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.LargeTests;
44  import org.apache.hadoop.hbase.TableName;
45  import org.apache.hadoop.hbase.TableNotFoundException;
46  import org.apache.hadoop.hbase.client.HBaseAdmin;
47  import org.apache.hadoop.hbase.client.HTable;
48  import org.apache.hadoop.hbase.client.ScannerCallable;
49  import org.apache.hadoop.hbase.ipc.RpcClient;
50  import org.apache.hadoop.hbase.ipc.RpcServer;
51  import org.apache.hadoop.hbase.master.HMaster;
52  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
53  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
54  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
55  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.util.FSUtils;
58  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
59  import org.apache.log4j.Level;
60  import org.junit.After;
61  import org.junit.AfterClass;
62  import org.junit.Before;
63  import org.junit.BeforeClass;
64  import org.junit.Test;
65  import org.junit.experimental.categories.Category;
66  
67  /**
68   * Test creating/using/deleting snapshots from the client
69   * <p>
70   * This is an end-to-end test for the snapshot utility
71   *
72   * TODO This is essentially a clone of TestSnapshotFromClient.  This is worth refactoring this
73   * because there will be a few more flavors of snapshots that need to run these tests.
74   */
75  @Category(LargeTests.class)
76  public class TestFlushSnapshotFromClient {
77    private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
78    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
79    private static final int NUM_RS = 2;
80    private static final String STRING_TABLE_NAME = "test";
81    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
82    private static final byte[] TEST_QUAL = Bytes.toBytes("q");
83    private static final TableName TABLE_NAME =
84        TableName.valueOf(STRING_TABLE_NAME);
85    private final int DEFAULT_NUM_ROWS = 100;
86  
87    /**
88     * Setup the config for the cluster
89     * @throws Exception on failure
90     */
91    @BeforeClass
92    public static void setupCluster() throws Exception {
93      ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
94      ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
95      ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
96      setupConf(UTIL.getConfiguration());
97      UTIL.startMiniCluster(NUM_RS);
98    }
99  
100   private static void setupConf(Configuration conf) {
101     // disable the ui
102     conf.setInt("hbase.regionsever.info.port", -1);
103     // change the flush size to a small amount, regulating number of store files
104     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
105     // so make sure we get a compaction when doing a load, but keep around some
106     // files in the store
107     conf.setInt("hbase.hstore.compaction.min", 10);
108     conf.setInt("hbase.hstore.compactionThreshold", 10);
109     // block writes if we get to 12 store files
110     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
111     // Enable snapshot
112     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
113     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
114       ConstantSizeRegionSplitPolicy.class.getName());
115   }
116 
117   @Before
118   public void setup() throws Exception {
119     SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
120   }
121 
122   @After
123   public void tearDown() throws Exception {
124     UTIL.deleteTable(TABLE_NAME);
125 
126     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
127     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
128   }
129 
130   @AfterClass
131   public static void cleanupTest() throws Exception {
132     try {
133       UTIL.shutdownMiniCluster();
134     } catch (Exception e) {
135       LOG.warn("failure shutting down cluster", e);
136     }
137   }
138 
139   /**
140    * Test simple flush snapshotting a table that is online
141    * @throws Exception
142    */
143   @Test (timeout=300000)
144   public void testFlushTableSnapshot() throws Exception {
145     HBaseAdmin admin = UTIL.getHBaseAdmin();
146     // make sure we don't fail on listing snapshots
147     SnapshotTestingUtils.assertNoSnapshots(admin);
148 
149     // put some stuff in the table
150     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
151     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
152 
153     LOG.debug("FS state before snapshot:");
154     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
155         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
156 
157     // take a snapshot of the enabled table
158     String snapshotString = "offlineTableSnapshot";
159     byte[] snapshot = Bytes.toBytes(snapshotString);
160     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
161     LOG.debug("Snapshot completed.");
162 
163     // make sure we have the snapshot
164     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
165       snapshot, TABLE_NAME);
166 
167     // make sure its a valid snapshot
168     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
169     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
170     LOG.debug("FS state after snapshot:");
171     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
172         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
173 
174     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
175         admin, fs);
176   }
177 
178    /**
179    * Test snapshotting a table that is online without flushing
180    * @throws Exception
181    */
182   @Test
183   public void testSkipFlushTableSnapshot() throws Exception {
184     HBaseAdmin admin = UTIL.getHBaseAdmin();
185     // make sure we don't fail on listing snapshots
186     SnapshotTestingUtils.assertNoSnapshots(admin);
187 
188     // put some stuff in the table
189     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
190     UTIL.loadTable(table, TEST_FAM);
191 
192     LOG.debug("FS state before snapshot:");
193     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
194         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
195 
196     // take a snapshot of the enabled table
197     String snapshotString = "skipFlushTableSnapshot";
198     byte[] snapshot = Bytes.toBytes(snapshotString);
199     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
200     LOG.debug("Snapshot completed.");
201 
202     // make sure we have the snapshot
203     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
204         snapshot, TABLE_NAME);
205 
206     // make sure its a valid snapshot
207     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
208     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
209     LOG.debug("FS state after snapshot:");
210     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
211         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
212 
213     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
214         admin, fs);
215 
216     admin.deleteSnapshot(snapshot);
217     snapshots = admin.listSnapshots();
218     SnapshotTestingUtils.assertNoSnapshots(admin);
219   }
220 
221 
222   /**
223    * Test simple flush snapshotting a table that is online
224    * @throws Exception
225    */
226   @Test (timeout=300000)
227   public void testFlushTableSnapshotWithProcedure() throws Exception {
228     HBaseAdmin admin = UTIL.getHBaseAdmin();
229     // make sure we don't fail on listing snapshots
230     SnapshotTestingUtils.assertNoSnapshots(admin);
231 
232     // put some stuff in the table
233     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
234     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
235 
236     LOG.debug("FS state before snapshot:");
237     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
238         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
239 
240     // take a snapshot of the enabled table
241     String snapshotString = "offlineTableSnapshot";
242     byte[] snapshot = Bytes.toBytes(snapshotString);
243     Map<String, String> props = new HashMap<String, String>();
244     props.put("table", STRING_TABLE_NAME);
245     admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
246         snapshotString, props);
247 
248 
249     LOG.debug("Snapshot completed.");
250 
251     // make sure we have the snapshot
252     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
253       snapshot, TABLE_NAME);
254 
255     // make sure its a valid snapshot
256     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
257     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
258     LOG.debug("FS state after snapshot:");
259     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
260         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
261 
262     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
263         admin, fs);
264   }
265 
266   @Test (timeout=300000)
267   public void testSnapshotFailsOnNonExistantTable() throws Exception {
268     HBaseAdmin admin = UTIL.getHBaseAdmin();
269     // make sure we don't fail on listing snapshots
270     SnapshotTestingUtils.assertNoSnapshots(admin);
271     String tableName = "_not_a_table";
272 
273     // make sure the table doesn't exist
274     boolean fail = false;
275     do {
276     try {
277       admin.getTableDescriptor(Bytes.toBytes(tableName));
278       fail = true;
279       LOG.error("Table:" + tableName + " already exists, checking a new name");
280       tableName = tableName+"!";
281     } catch (TableNotFoundException e) {
282       fail = false;
283       }
284     } while (fail);
285 
286     // snapshot the non-existant table
287     try {
288       admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
289       fail("Snapshot succeeded even though there is not table.");
290     } catch (SnapshotCreationException e) {
291       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
292     }
293   }
294 
295   @Test(timeout = 300000)
296   public void testAsyncFlushSnapshot() throws Exception {
297     HBaseAdmin admin = UTIL.getHBaseAdmin();
298     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
299         .setTable(TABLE_NAME.getNameAsString())
300         .setType(SnapshotDescription.Type.FLUSH)
301         .build();
302 
303     // take the snapshot async
304     admin.takeSnapshotAsync(snapshot);
305 
306     // constantly loop, looking for the snapshot to complete
307     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
308     SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
309     LOG.info(" === Async Snapshot Completed ===");
310     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
311       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
312     // make sure we get the snapshot
313     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
314   }
315 
316   @Test (timeout=300000)
317   public void testSnapshotStateAfterMerge() throws Exception {
318     int numRows = DEFAULT_NUM_ROWS;
319     HBaseAdmin admin = UTIL.getHBaseAdmin();
320     // make sure we don't fail on listing snapshots
321     SnapshotTestingUtils.assertNoSnapshots(admin);
322     // load the table so we have some data
323     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
324 
325     // Take a snapshot
326     String snapshotBeforeMergeName = "snapshotBeforeMerge";
327     admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
328 
329     // Clone the table
330     String cloneBeforeMergeName = "cloneBeforeMerge";
331     admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
332     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
333 
334     // Merge two regions
335     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
336     Collections.sort(regions, new Comparator<HRegionInfo>() {
337       public int compare(HRegionInfo r1, HRegionInfo r2) {
338         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
339       }
340     });
341 
342     int numRegions = admin.getTableRegions(TABLE_NAME).size();
343     int numRegionsAfterMerge = numRegions - 2;
344     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
345         regions.get(2).getEncodedNameAsBytes(), true);
346     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
347         regions.get(6).getEncodedNameAsBytes(), true);
348 
349     // Verify that there's one region less
350     waitRegionsAfterMerge(numRegionsAfterMerge);
351     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
352 
353     // Clone the table
354     String cloneAfterMergeName = "cloneAfterMerge";
355     admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
356     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
357 
358     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
359     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
360     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
361 
362     // test that we can delete the snapshot
363     UTIL.deleteTable(cloneAfterMergeName);
364     UTIL.deleteTable(cloneBeforeMergeName);
365   }
366 
367   @Test (timeout=300000)
368   public void testTakeSnapshotAfterMerge() throws Exception {
369     int numRows = DEFAULT_NUM_ROWS;
370     HBaseAdmin admin = UTIL.getHBaseAdmin();
371     // make sure we don't fail on listing snapshots
372     SnapshotTestingUtils.assertNoSnapshots(admin);
373     // load the table so we have some data
374     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
375 
376     // Merge two regions
377     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
378     Collections.sort(regions, new Comparator<HRegionInfo>() {
379       public int compare(HRegionInfo r1, HRegionInfo r2) {
380         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
381       }
382     });
383 
384     int numRegions = admin.getTableRegions(TABLE_NAME).size();
385     int numRegionsAfterMerge = numRegions - 2;
386     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
387         regions.get(2).getEncodedNameAsBytes(), true);
388     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
389         regions.get(6).getEncodedNameAsBytes(), true);
390 
391     waitRegionsAfterMerge(numRegionsAfterMerge);
392     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
393 
394     // Take a snapshot
395     String snapshotName = "snapshotAfterMerge";
396     SnapshotTestingUtils.snapshot(admin, snapshotName, STRING_TABLE_NAME,
397       SnapshotDescription.Type.FLUSH, 3);
398 
399     // Clone the table
400     String cloneName = "cloneMerge";
401     admin.cloneSnapshot(snapshotName, cloneName);
402     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
403 
404     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
405     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
406 
407     // test that we can delete the snapshot
408     UTIL.deleteTable(cloneName);
409   }
410 
411   /**
412    * Basic end-to-end test of simple-flush-based snapshots
413    */
414   @Test (timeout=300000)
415   public void testFlushCreateListDestroy() throws Exception {
416     LOG.debug("------- Starting Snapshot test -------------");
417     HBaseAdmin admin = UTIL.getHBaseAdmin();
418     // make sure we don't fail on listing snapshots
419     SnapshotTestingUtils.assertNoSnapshots(admin);
420     // load the table so we have some data
421     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
422 
423     String snapshotName = "flushSnapshotCreateListDestroy";
424     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
425     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
426     SnapshotTestingUtils.createSnapshotAndValidate(admin,
427       TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
428       snapshotName, rootDir, fs, true);
429   }
430 
431   /**
432    * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
433    * same table currently running and that concurrent snapshots on different tables can both
434    * succeed concurretly.
435    */
436   @Test(timeout=300000)
437   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
438     final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
439     final TableName TABLE2_NAME =
440         TableName.valueOf(STRING_TABLE2_NAME);
441 
442     int ssNum = 20;
443     HBaseAdmin admin = UTIL.getHBaseAdmin();
444     // make sure we don't fail on listing snapshots
445     SnapshotTestingUtils.assertNoSnapshots(admin);
446     // create second testing table
447     SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
448     // load the table so we have some data
449     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
450     SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
451 
452     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
453     // We'll have one of these per thread
454     class SSRunnable implements Runnable {
455       SnapshotDescription ss;
456       SSRunnable(SnapshotDescription ss) {
457         this.ss = ss;
458       }
459 
460       @Override
461       public void run() {
462         try {
463           HBaseAdmin admin = UTIL.getHBaseAdmin();
464           LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
465           admin.takeSnapshotAsync(ss);
466         } catch (Exception e) {
467           LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
468               ss)
469               + ".  This is ok, we expect some", e);
470         }
471         LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
472         toBeSubmitted.countDown();
473       }
474     };
475 
476     // build descriptions
477     SnapshotDescription[] descs = new SnapshotDescription[ssNum];
478     for (int i = 0; i < ssNum; i++) {
479       SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
480       builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
481       builder.setName("ss"+i);
482       builder.setType(SnapshotDescription.Type.FLUSH);
483       descs[i] = builder.build();
484     }
485 
486     // kick each off its own thread
487     for (int i=0 ; i < ssNum; i++) {
488       new Thread(new SSRunnable(descs[i])).start();
489     }
490 
491     // wait until all have been submitted
492     toBeSubmitted.await();
493 
494     // loop until all are done.
495     while (true) {
496       int doneCount = 0;
497       for (SnapshotDescription ss : descs) {
498         try {
499           if (admin.isSnapshotFinished(ss)) {
500             doneCount++;
501           }
502         } catch (Exception e) {
503           LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
504           doneCount++;
505         }
506       }
507       if (doneCount == descs.length) {
508         break;
509       }
510       Thread.sleep(100);
511     }
512 
513     // dump for debugging
514     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
515 
516     List<SnapshotDescription> taken = admin.listSnapshots();
517     int takenSize = taken.size();
518     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
519     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
520         " issued many requests", takenSize < ssNum && takenSize > 0);
521 
522     // Verify that there's at least one snapshot per table
523     int t1SnapshotsCount = 0;
524     int t2SnapshotsCount = 0;
525     for (SnapshotDescription ss : taken) {
526       if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
527         t1SnapshotsCount++;
528       } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
529         t2SnapshotsCount++;
530       }
531     }
532     assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
533     assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
534 
535     UTIL.deleteTable(TABLE2_NAME);
536   }
537 
538   private void logFSTree(Path root) throws IOException {
539     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
540   }
541 
542   private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
543       throws IOException, InterruptedException {
544     HBaseAdmin admin = UTIL.getHBaseAdmin();
545     // Verify that there's one region less
546     long startTime = System.currentTimeMillis();
547     while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
548       // This may be flaky... if after 15sec the merge is not complete give up
549       // it will fail in the assertEquals(numRegionsAfterMerge).
550       if ((System.currentTimeMillis() - startTime) > 15000)
551         break;
552       Thread.sleep(100);
553     }
554     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
555   }
556 }