View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.HashMap;
28  import java.util.HashSet;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Set;
32  import java.util.concurrent.CountDownLatch;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.commons.logging.impl.Log4JLogger;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.LargeTests;
44  import org.apache.hadoop.hbase.TableName;
45  import org.apache.hadoop.hbase.TableNotFoundException;
46  import org.apache.hadoop.hbase.client.HBaseAdmin;
47  import org.apache.hadoop.hbase.client.HTable;
48  import org.apache.hadoop.hbase.client.ScannerCallable;
49  import org.apache.hadoop.hbase.ipc.RpcClient;
50  import org.apache.hadoop.hbase.ipc.RpcServer;
51  import org.apache.hadoop.hbase.master.HMaster;
52  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
53  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
54  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
55  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.util.FSUtils;
58  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
59  import org.apache.log4j.Level;
60  import org.junit.After;
61  import org.junit.AfterClass;
62  import org.junit.Before;
63  import org.junit.BeforeClass;
64  import org.junit.Test;
65  import org.junit.experimental.categories.Category;
66  
67  /**
68   * Test creating/using/deleting snapshots from the client
69   * <p>
70   * This is an end-to-end test for the snapshot utility
71   *
72   * TODO This is essentially a clone of TestSnapshotFromClient.  This is worth refactoring this
73   * because there will be a few more flavors of snapshots that need to run these tests.
74   */
75  @Category(LargeTests.class)
76  public class TestFlushSnapshotFromClient {
77    private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
78    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
79    private static final int NUM_RS = 2;
80    private static final String STRING_TABLE_NAME = "test";
81    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
82    private static final byte[] TEST_QUAL = Bytes.toBytes("q");
83    private static final TableName TABLE_NAME =
84        TableName.valueOf(STRING_TABLE_NAME);
85    private final int DEFAULT_NUM_ROWS = 100;
86  
87    /**
88     * Setup the config for the cluster
89     * @throws Exception on failure
90     */
91    @BeforeClass
92    public static void setupCluster() throws Exception {
93      // Uncomment the following lines if more verbosity is needed for
94      // debugging (see HBASE-12285 for details).
95      //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
96      //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
97      //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
98      setupConf(UTIL.getConfiguration());
99      UTIL.startMiniCluster(NUM_RS);
100   }
101 
102   private static void setupConf(Configuration conf) {
103     // disable the ui
104     conf.setInt("hbase.regionsever.info.port", -1);
105     // change the flush size to a small amount, regulating number of store files
106     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
107     // so make sure we get a compaction when doing a load, but keep around some
108     // files in the store
109     conf.setInt("hbase.hstore.compaction.min", 10);
110     conf.setInt("hbase.hstore.compactionThreshold", 10);
111     // block writes if we get to 12 store files
112     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
113     // Enable snapshot
114     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
115     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
116       ConstantSizeRegionSplitPolicy.class.getName());
117   }
118 
119   @Before
120   public void setup() throws Exception {
121     SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
122   }
123 
124   @After
125   public void tearDown() throws Exception {
126     UTIL.deleteTable(TABLE_NAME);
127 
128     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
129     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
130   }
131 
132   @AfterClass
133   public static void cleanupTest() throws Exception {
134     try {
135       UTIL.shutdownMiniCluster();
136     } catch (Exception e) {
137       LOG.warn("failure shutting down cluster", e);
138     }
139   }
140 
141   /**
142    * Test simple flush snapshotting a table that is online
143    * @throws Exception
144    */
145   @Test (timeout=300000)
146   public void testFlushTableSnapshot() throws Exception {
147     HBaseAdmin admin = UTIL.getHBaseAdmin();
148     // make sure we don't fail on listing snapshots
149     SnapshotTestingUtils.assertNoSnapshots(admin);
150 
151     // put some stuff in the table
152     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
153     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
154 
155     LOG.debug("FS state before snapshot:");
156     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
157         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
158 
159     // take a snapshot of the enabled table
160     String snapshotString = "offlineTableSnapshot";
161     byte[] snapshot = Bytes.toBytes(snapshotString);
162     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
163     LOG.debug("Snapshot completed.");
164 
165     // make sure we have the snapshot
166     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
167       snapshot, TABLE_NAME);
168 
169     // make sure its a valid snapshot
170     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
171     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
172     LOG.debug("FS state after snapshot:");
173     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
174         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
175 
176     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
177         admin, fs);
178   }
179 
180    /**
181    * Test snapshotting a table that is online without flushing
182    * @throws Exception
183    */
184   @Test
185   public void testSkipFlushTableSnapshot() throws Exception {
186     HBaseAdmin admin = UTIL.getHBaseAdmin();
187     // make sure we don't fail on listing snapshots
188     SnapshotTestingUtils.assertNoSnapshots(admin);
189 
190     // put some stuff in the table
191     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
192     UTIL.loadTable(table, TEST_FAM);
193 
194     LOG.debug("FS state before snapshot:");
195     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
196         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
197 
198     // take a snapshot of the enabled table
199     String snapshotString = "skipFlushTableSnapshot";
200     byte[] snapshot = Bytes.toBytes(snapshotString);
201     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
202     LOG.debug("Snapshot completed.");
203 
204     // make sure we have the snapshot
205     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
206         snapshot, TABLE_NAME);
207 
208     // make sure its a valid snapshot
209     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
210     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
211     LOG.debug("FS state after snapshot:");
212     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
213         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
214 
215     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
216         admin, fs);
217 
218     admin.deleteSnapshot(snapshot);
219     snapshots = admin.listSnapshots();
220     SnapshotTestingUtils.assertNoSnapshots(admin);
221   }
222 
223 
224   /**
225    * Test simple flush snapshotting a table that is online
226    * @throws Exception
227    */
228   @Test (timeout=300000)
229   public void testFlushTableSnapshotWithProcedure() throws Exception {
230     HBaseAdmin admin = UTIL.getHBaseAdmin();
231     // make sure we don't fail on listing snapshots
232     SnapshotTestingUtils.assertNoSnapshots(admin);
233 
234     // put some stuff in the table
235     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
236     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
237 
238     LOG.debug("FS state before snapshot:");
239     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
240         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
241 
242     // take a snapshot of the enabled table
243     String snapshotString = "offlineTableSnapshot";
244     byte[] snapshot = Bytes.toBytes(snapshotString);
245     Map<String, String> props = new HashMap<String, String>();
246     props.put("table", STRING_TABLE_NAME);
247     admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
248         snapshotString, props);
249 
250 
251     LOG.debug("Snapshot completed.");
252 
253     // make sure we have the snapshot
254     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
255       snapshot, TABLE_NAME);
256 
257     // make sure its a valid snapshot
258     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
259     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
260     LOG.debug("FS state after snapshot:");
261     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
262         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
263 
264     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
265         admin, fs);
266   }
267 
268   @Test (timeout=300000)
269   public void testSnapshotFailsOnNonExistantTable() throws Exception {
270     HBaseAdmin admin = UTIL.getHBaseAdmin();
271     // make sure we don't fail on listing snapshots
272     SnapshotTestingUtils.assertNoSnapshots(admin);
273     String tableName = "_not_a_table";
274 
275     // make sure the table doesn't exist
276     boolean fail = false;
277     do {
278     try {
279       admin.getTableDescriptor(Bytes.toBytes(tableName));
280       fail = true;
281       LOG.error("Table:" + tableName + " already exists, checking a new name");
282       tableName = tableName+"!";
283     } catch (TableNotFoundException e) {
284       fail = false;
285       }
286     } while (fail);
287 
288     // snapshot the non-existant table
289     try {
290       admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
291       fail("Snapshot succeeded even though there is not table.");
292     } catch (SnapshotCreationException e) {
293       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
294     }
295   }
296 
297   @Test(timeout = 300000)
298   public void testAsyncFlushSnapshot() throws Exception {
299     HBaseAdmin admin = UTIL.getHBaseAdmin();
300     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
301         .setTable(TABLE_NAME.getNameAsString())
302         .setType(SnapshotDescription.Type.FLUSH)
303         .build();
304 
305     // take the snapshot async
306     admin.takeSnapshotAsync(snapshot);
307 
308     // constantly loop, looking for the snapshot to complete
309     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
310     SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
311     LOG.info(" === Async Snapshot Completed ===");
312     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
313       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
314     // make sure we get the snapshot
315     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
316   }
317 
318   @Test (timeout=300000)
319   public void testSnapshotStateAfterMerge() throws Exception {
320     int numRows = DEFAULT_NUM_ROWS;
321     HBaseAdmin admin = UTIL.getHBaseAdmin();
322     // make sure we don't fail on listing snapshots
323     SnapshotTestingUtils.assertNoSnapshots(admin);
324     // load the table so we have some data
325     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
326 
327     // Take a snapshot
328     String snapshotBeforeMergeName = "snapshotBeforeMerge";
329     admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
330 
331     // Clone the table
332     String cloneBeforeMergeName = "cloneBeforeMerge";
333     admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
334     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
335 
336     // Merge two regions
337     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
338     Collections.sort(regions, new Comparator<HRegionInfo>() {
339       public int compare(HRegionInfo r1, HRegionInfo r2) {
340         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
341       }
342     });
343 
344     int numRegions = admin.getTableRegions(TABLE_NAME).size();
345     int numRegionsAfterMerge = numRegions - 2;
346     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
347         regions.get(2).getEncodedNameAsBytes(), true);
348     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
349         regions.get(6).getEncodedNameAsBytes(), true);
350 
351     // Verify that there's one region less
352     waitRegionsAfterMerge(numRegionsAfterMerge);
353     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
354 
355     // Clone the table
356     String cloneAfterMergeName = "cloneAfterMerge";
357     admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
358     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
359 
360     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
361     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
362     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
363 
364     // test that we can delete the snapshot
365     UTIL.deleteTable(cloneAfterMergeName);
366     UTIL.deleteTable(cloneBeforeMergeName);
367   }
368 
369   @Test (timeout=300000)
370   public void testTakeSnapshotAfterMerge() throws Exception {
371     int numRows = DEFAULT_NUM_ROWS;
372     HBaseAdmin admin = UTIL.getHBaseAdmin();
373     // make sure we don't fail on listing snapshots
374     SnapshotTestingUtils.assertNoSnapshots(admin);
375     // load the table so we have some data
376     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
377 
378     // Merge two regions
379     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
380     Collections.sort(regions, new Comparator<HRegionInfo>() {
381       public int compare(HRegionInfo r1, HRegionInfo r2) {
382         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
383       }
384     });
385 
386     int numRegions = admin.getTableRegions(TABLE_NAME).size();
387     int numRegionsAfterMerge = numRegions - 2;
388     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
389         regions.get(2).getEncodedNameAsBytes(), true);
390     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
391         regions.get(6).getEncodedNameAsBytes(), true);
392 
393     waitRegionsAfterMerge(numRegionsAfterMerge);
394     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
395 
396     // Take a snapshot
397     String snapshotName = "snapshotAfterMerge";
398     SnapshotTestingUtils.snapshot(admin, snapshotName, STRING_TABLE_NAME,
399       SnapshotDescription.Type.FLUSH, 3);
400 
401     // Clone the table
402     String cloneName = "cloneMerge";
403     admin.cloneSnapshot(snapshotName, cloneName);
404     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
405 
406     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
407     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
408 
409     // test that we can delete the snapshot
410     UTIL.deleteTable(cloneName);
411   }
412 
413   /**
414    * Basic end-to-end test of simple-flush-based snapshots
415    */
416   @Test (timeout=300000)
417   public void testFlushCreateListDestroy() throws Exception {
418     LOG.debug("------- Starting Snapshot test -------------");
419     HBaseAdmin admin = UTIL.getHBaseAdmin();
420     // make sure we don't fail on listing snapshots
421     SnapshotTestingUtils.assertNoSnapshots(admin);
422     // load the table so we have some data
423     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
424 
425     String snapshotName = "flushSnapshotCreateListDestroy";
426     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
427     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
428     SnapshotTestingUtils.createSnapshotAndValidate(admin,
429       TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
430       snapshotName, rootDir, fs, true);
431   }
432 
433   /**
434    * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
435    * same table currently running and that concurrent snapshots on different tables can both
436    * succeed concurretly.
437    */
438   @Test(timeout=300000)
439   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
440     final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
441     final TableName TABLE2_NAME =
442         TableName.valueOf(STRING_TABLE2_NAME);
443 
444     int ssNum = 20;
445     HBaseAdmin admin = UTIL.getHBaseAdmin();
446     // make sure we don't fail on listing snapshots
447     SnapshotTestingUtils.assertNoSnapshots(admin);
448     // create second testing table
449     SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
450     // load the table so we have some data
451     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
452     SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
453 
454     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
455     // We'll have one of these per thread
456     class SSRunnable implements Runnable {
457       SnapshotDescription ss;
458       SSRunnable(SnapshotDescription ss) {
459         this.ss = ss;
460       }
461 
462       @Override
463       public void run() {
464         try {
465           HBaseAdmin admin = UTIL.getHBaseAdmin();
466           LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
467           admin.takeSnapshotAsync(ss);
468         } catch (Exception e) {
469           LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
470               ss)
471               + ".  This is ok, we expect some", e);
472         }
473         LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
474         toBeSubmitted.countDown();
475       }
476     };
477 
478     // build descriptions
479     SnapshotDescription[] descs = new SnapshotDescription[ssNum];
480     for (int i = 0; i < ssNum; i++) {
481       SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
482       builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
483       builder.setName("ss"+i);
484       builder.setType(SnapshotDescription.Type.FLUSH);
485       descs[i] = builder.build();
486     }
487 
488     // kick each off its own thread
489     for (int i=0 ; i < ssNum; i++) {
490       new Thread(new SSRunnable(descs[i])).start();
491     }
492 
493     // wait until all have been submitted
494     toBeSubmitted.await();
495 
496     // loop until all are done.
497     while (true) {
498       int doneCount = 0;
499       for (SnapshotDescription ss : descs) {
500         try {
501           if (admin.isSnapshotFinished(ss)) {
502             doneCount++;
503           }
504         } catch (Exception e) {
505           LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
506           doneCount++;
507         }
508       }
509       if (doneCount == descs.length) {
510         break;
511       }
512       Thread.sleep(100);
513     }
514 
515     // dump for debugging
516     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
517 
518     List<SnapshotDescription> taken = admin.listSnapshots();
519     int takenSize = taken.size();
520     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
521     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
522         " issued many requests", takenSize < ssNum && takenSize > 0);
523 
524     // Verify that there's at least one snapshot per table
525     int t1SnapshotsCount = 0;
526     int t2SnapshotsCount = 0;
527     for (SnapshotDescription ss : taken) {
528       if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
529         t1SnapshotsCount++;
530       } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
531         t2SnapshotsCount++;
532       }
533     }
534     assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
535     assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
536 
537     UTIL.deleteTable(TABLE2_NAME);
538   }
539 
540   private void logFSTree(Path root) throws IOException {
541     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
542   }
543 
544   private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
545       throws IOException, InterruptedException {
546     HBaseAdmin admin = UTIL.getHBaseAdmin();
547     // Verify that there's one region less
548     long startTime = System.currentTimeMillis();
549     while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
550       // This may be flaky... if after 15sec the merge is not complete give up
551       // it will fail in the assertEquals(numRegionsAfterMerge).
552       if ((System.currentTimeMillis() - startTime) > 15000)
553         break;
554       Thread.sleep(100);
555     }
556     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
557   }
558 }