View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.Arrays;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Set;
32  import java.util.HashSet;
33  import java.util.TreeSet;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FSDataOutputStream;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.fs.PathFilter;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.TableNotEnabledException;
49  import org.apache.hadoop.hbase.client.Durability;
50  import org.apache.hadoop.hbase.client.HBaseAdmin;
51  import org.apache.hadoop.hbase.client.HTable;
52  import org.apache.hadoop.hbase.client.Put;
53  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
54  import org.apache.hadoop.hbase.io.HFileLink;
55  import org.apache.hadoop.hbase.master.HMaster;
56  import org.apache.hadoop.hbase.master.MasterFileSystem;
57  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
58  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
59  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
60  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
61  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
62  import org.apache.hadoop.hbase.regionserver.HRegion;
63  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
64  import org.apache.hadoop.hbase.regionserver.HRegionServer;
65  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
66  import org.apache.hadoop.hbase.util.Bytes;
67  import org.apache.hadoop.hbase.util.FSTableDescriptors;
68  import org.apache.hadoop.hbase.util.FSUtils;
69  import org.apache.hadoop.hbase.util.FSVisitor;
70  import org.apache.hadoop.hbase.util.FSUtils;
71  import org.apache.hadoop.hbase.util.MD5Hash;
72  import org.junit.Assert;
73  
74  import com.google.protobuf.ServiceException;
75  
76  /**
77   * Utilities class for snapshots
78   */
79  public class SnapshotTestingUtils {
80  
81    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
82    private static byte[] KEYS = Bytes.toBytes("0123456789");
83  
84    /**
85     * Assert that we don't have any snapshots lists
86     *
87     * @throws IOException
88     *           if the admin operation fails
89     */
90    public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
91      assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
92          .size());
93    }
94  
95    /**
96     * Make sure that there is only one snapshot returned from the master and its
97     * name and table match the passed in parameters.
98     */
99    public static List<SnapshotDescription> assertExistsMatchingSnapshot(
100       HBaseAdmin admin, String snapshotName, TableName tableName)
101       throws IOException {
102     // list the snapshot
103     List<SnapshotDescription> snapshots = admin.listSnapshots();
104 
105     List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
106     for (SnapshotDescription sd : snapshots) {
107       if (snapshotName.equals(sd.getName()) &&
108           tableName.equals(TableName.valueOf(sd.getTable()))) {
109         returnedSnapshots.add(sd);
110       }
111     }
112 
113     Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
114     return returnedSnapshots;
115   }
116 
117   /**
118    * Make sure that there is only one snapshot returned from the master
119    */
120   public static void assertOneSnapshotThatMatches(HBaseAdmin admin,
121       SnapshotDescription snapshot) throws IOException {
122     assertOneSnapshotThatMatches(admin, snapshot.getName(),
123         TableName.valueOf(snapshot.getTable()));
124   }
125 
126   /**
127    * Make sure that there is only one snapshot returned from the master and its
128    * name and table match the passed in parameters.
129    */
130   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
131       HBaseAdmin admin, String snapshotName, TableName tableName)
132       throws IOException {
133     // list the snapshot
134     List<SnapshotDescription> snapshots = admin.listSnapshots();
135 
136     assertEquals("Should only have 1 snapshot", 1, snapshots.size());
137     assertEquals(snapshotName, snapshots.get(0).getName());
138     assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
139 
140     return snapshots;
141   }
142 
143   /**
144    * Make sure that there is only one snapshot returned from the master and its
145    * name and table match the passed in parameters.
146    */
147   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
148       HBaseAdmin admin, byte[] snapshot, TableName tableName) throws IOException {
149     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
150         tableName);
151   }
152 
153   /**
154    * Confirm that the snapshot contains references to all the files that should
155    * be in the snapshot.
156    */
157   public static void confirmSnapshotValid(
158       SnapshotDescription snapshotDescriptor, TableName tableName,
159       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
160       throws IOException {
161     ArrayList nonEmptyTestFamilies = new ArrayList(1);
162     nonEmptyTestFamilies.add(testFamily);
163     confirmSnapshotValid(snapshotDescriptor, tableName,
164       nonEmptyTestFamilies, null, rootDir, admin, fs);
165   }
166 
167   /**
168    * Confirm that the snapshot has no references files but only metadata.
169    */
170   public static void confirmEmptySnapshotValid(
171       SnapshotDescription snapshotDescriptor, TableName tableName,
172       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
173       throws IOException {
174     ArrayList emptyTestFamilies = new ArrayList(1);
175     emptyTestFamilies.add(testFamily);
176     confirmSnapshotValid(snapshotDescriptor, tableName,
177       null, emptyTestFamilies, rootDir, admin, fs);
178   }
179 
180   /**
181    * Confirm that the snapshot contains references to all the files that should
182    * be in the snapshot. This method also perform some redundant check like
183    * the existence of the snapshotinfo or the regioninfo which are done always
184    * by the MasterSnapshotVerifier, at the end of the snapshot operation.
185    */
186   public static void confirmSnapshotValid(
187       SnapshotDescription snapshotDescriptor, TableName tableName,
188       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
189       Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException {
190     final Configuration conf = admin.getConfiguration();
191 
192     // check snapshot dir
193     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
194         snapshotDescriptor, rootDir);
195     assertTrue(fs.exists(snapshotDir));
196 
197     SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
198 
199     // Extract regions and families with store files
200     final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
201 
202     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
203     Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
204     for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
205       SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
206           new SnapshotReferenceUtil.StoreFileVisitor() {
207         @Override
208         public void storeFile(final HRegionInfo regionInfo, final String family,
209               final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
210           snapshotFamilies.add(Bytes.toBytes(family));
211         }
212       });
213     }
214 
215     // Verify that there are store files in the specified families
216     if (nonEmptyTestFamilies != null) {
217       for (final byte[] familyName: nonEmptyTestFamilies) {
218         assertTrue(snapshotFamilies.contains(familyName));
219       }
220     }
221 
222     // Verify that there are no store files in the specified families
223     if (emptyTestFamilies != null) {
224       for (final byte[] familyName: emptyTestFamilies) {
225         assertFalse(snapshotFamilies.contains(familyName));
226       }
227     }
228 
229     // check the region snapshot for all the regions
230     List<HRegionInfo> regions = admin.getTableRegions(tableName);
231     assertEquals(regions.size(), regionManifests.size());
232 
233     // Verify Regions (redundant check, see MasterSnapshotVerifier)
234     for (HRegionInfo info : regions) {
235       String regionName = info.getEncodedName();
236       assertTrue(regionManifests.containsKey(regionName));
237     }
238   }
239 
240   /**
241    * Helper method for testing async snapshot operations. Just waits for the
242    * given snapshot to complete on the server by repeatedly checking the master.
243    *
244    * @param master: the master running the snapshot
245    * @param snapshot: the snapshot to check
246    * @param sleep: amount to sleep between checks to see if the snapshot is done
247    * @throws ServiceException if the snapshot fails
248    */
249   public static void waitForSnapshotToComplete(HMaster master,
250       SnapshotDescription snapshot, long sleep) throws ServiceException {
251     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
252         .setSnapshot(snapshot).build();
253     IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
254         .buildPartial();
255     while (!done.getDone()) {
256       done = master.isSnapshotDone(null, request);
257       try {
258         Thread.sleep(sleep);
259       } catch (InterruptedException e) {
260         throw new ServiceException(e);
261       }
262     }
263   }
264 
265   /*
266    * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
267    * except for the last CorruptedSnapshotException
268    */
269   public static void snapshot(HBaseAdmin admin,
270       final String snapshotName, final String tableName,
271       SnapshotDescription.Type type, int numTries) throws IOException {
272     int tries = 0;
273     CorruptedSnapshotException lastEx = null;
274     while (tries++ < numTries) {
275       try {
276         admin.snapshot(snapshotName, tableName, type);
277         return;
278       } catch (CorruptedSnapshotException cse) {
279         LOG.warn("Got CorruptedSnapshotException", cse);
280         lastEx = cse;
281       }
282     }
283     throw lastEx;
284   }
285 
286   public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName)
287       throws IOException {
288     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
289   }
290 
291   public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName)
292       throws IOException {
293     // delete the taken snapshot
294     admin.deleteSnapshot(snapshotName);
295     assertNoSnapshots(admin);
296   }
297 
298   /**
299    * Expect the snapshot to throw an error when checking if the snapshot is
300    * complete
301    *
302    * @param master master to check
303    * @param snapshot the {@link SnapshotDescription} request to pass to the master
304    * @param clazz expected exception from the master
305    */
306   public static void expectSnapshotDoneException(HMaster master,
307       IsSnapshotDoneRequest snapshot,
308       Class<? extends HBaseSnapshotException> clazz) {
309     try {
310       master.isSnapshotDone(null, snapshot);
311       Assert.fail("didn't fail to lookup a snapshot");
312     } catch (ServiceException se) {
313       try {
314         throw ProtobufUtil.getRemoteException(se);
315       } catch (HBaseSnapshotException e) {
316         assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
317       } catch (Throwable t) {
318         Assert.fail("Threw an unexpected exception:" + t);
319       }
320     }
321   }
322 
323   /**
324    * List all the HFiles in the given table
325    *
326    * @param fs: FileSystem where the table lives
327    * @param tableDir directory of the table
328    * @return array of the current HFiles in the table (could be a zero-length array)
329    * @throws IOException on unexecpted error reading the FS
330    */
331   public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
332       throws IOException {
333     final ArrayList<Path> hfiles = new ArrayList<Path>();
334     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
335       @Override
336       public void storeFile(final String region, final String family, final String hfileName)
337           throws IOException {
338         hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
339       }
340     });
341     return hfiles.toArray(new Path[hfiles.size()]);
342   }
343 
344   public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
345       throws IOException {
346     Path[] files = listHFiles(fs, tableDir);
347     String[] names = new String[files.length];
348     for (int i = 0; i < files.length; ++i) {
349       names[i] = files[i].getName();
350     }
351     Arrays.sort(names);
352     return names;
353   }
354 
355   /**
356    * Take a snapshot of the specified table and verify that the given family is
357    * not empty. Note that this will leave the table disabled
358    * in the case of an offline snapshot.
359    */
360   public static void createSnapshotAndValidate(HBaseAdmin admin,
361       TableName tableName, String familyName, String snapshotNameString,
362       Path rootDir, FileSystem fs, boolean onlineSnapshot)
363       throws Exception {
364     ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
365     nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
366     createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
367                               snapshotNameString, rootDir, fs, onlineSnapshot);
368   }
369 
370   /**
371    * Take a snapshot of the specified table and verify the given families.
372    * Note that this will leave the table disabled in the case of an offline snapshot.
373    */
374   public static void createSnapshotAndValidate(HBaseAdmin admin,
375       TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
376       String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
377         throws Exception {
378     if (!onlineSnapshot) {
379       try {
380         admin.disableTable(tableName);
381       } catch (TableNotEnabledException tne) {
382         LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
383             "already disabled.");
384       }
385     }
386     admin.snapshot(snapshotNameString, tableName);
387 
388     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
389       snapshotNameString, tableName);
390     if (snapshots == null || snapshots.size() != 1) {
391       Assert.fail("Incorrect number of snapshots for table " + tableName);
392     }
393 
394     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
395       emptyFamilyNames, rootDir, admin, fs);
396   }
397 
398   /**
399    * Corrupt the specified snapshot by deleting some files.
400    *
401    * @param util {@link HBaseTestingUtility}
402    * @param snapshotName name of the snapshot to corrupt
403    * @return array of the corrupted HFiles
404    * @throws IOException on unexecpted error reading the FS
405    */
406   public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
407       throws IOException {
408     final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
409     final FileSystem fs = mfs.getFileSystem();
410 
411     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
412                                                                         mfs.getRootDir());
413     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
414     final TableName table = TableName.valueOf(snapshotDesc.getTable());
415 
416     final ArrayList corruptedFiles = new ArrayList();
417     final Configuration conf = util.getConfiguration();
418     SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
419         new SnapshotReferenceUtil.StoreFileVisitor() {
420       @Override
421       public void storeFile(final HRegionInfo regionInfo, final String family,
422             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
423         String region = regionInfo.getEncodedName();
424         String hfile = storeFile.getName();
425         HFileLink link = HFileLink.create(conf, table, region, family, hfile);
426         if (corruptedFiles.size() % 2 == 0) {
427           fs.delete(link.getAvailablePath(fs));
428           corruptedFiles.add(hfile);
429         }
430       }
431     });
432 
433     assertTrue(corruptedFiles.size() > 0);
434     return corruptedFiles;
435   }
436 
437   // ==========================================================================
438   //  Snapshot Mock
439   // ==========================================================================
440   public static class SnapshotMock {
441     private final static String TEST_FAMILY = "cf";
442     public final static int TEST_NUM_REGIONS = 4;
443 
444     private final Configuration conf;
445     private final FileSystem fs;
446     private final Path rootDir;
447 
448     static class RegionData {
449       public HRegionInfo hri;
450       public Path tableDir;
451       public Path[] files;
452 
453       public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
454         this.tableDir = tableDir;
455         this.hri = hri;
456         this.files = new Path[nfiles];
457       }
458     }
459 
460     public static class SnapshotBuilder {
461       private final RegionData[] tableRegions;
462       private final SnapshotDescription desc;
463       private final HTableDescriptor htd;
464       private final Configuration conf;
465       private final FileSystem fs;
466       private final Path rootDir;
467       private Path snapshotDir;
468       private int snapshotted = 0;
469 
470       public SnapshotBuilder(final Configuration conf, final FileSystem fs,
471           final Path rootDir, final HTableDescriptor htd,
472           final SnapshotDescription desc, final RegionData[] tableRegions)
473           throws IOException {
474         this.fs = fs;
475         this.conf = conf;
476         this.rootDir = rootDir;
477         this.htd = htd;
478         this.desc = desc;
479         this.tableRegions = tableRegions;
480         this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
481         new FSTableDescriptors(conf)
482           .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
483       }
484 
485       public HTableDescriptor getTableDescriptor() {
486         return this.htd;
487       }
488 
489       public SnapshotDescription getSnapshotDescription() {
490         return this.desc;
491       }
492 
493       public Path getSnapshotsDir() {
494         return this.snapshotDir;
495       }
496 
497       public Path[] addRegion() throws IOException {
498         return addRegion(desc);
499       }
500 
501       public Path[] addRegionV1() throws IOException {
502         return addRegion(desc.toBuilder()
503                           .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
504                           .build());
505       }
506 
507       public Path[] addRegionV2() throws IOException {
508         return addRegion(desc.toBuilder()
509                           .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
510                           .build());
511       }
512 
513       private Path[] addRegion(final SnapshotDescription desc) throws IOException {
514         if (this.snapshotted == tableRegions.length) {
515           throw new UnsupportedOperationException("No more regions in the table");
516         }
517 
518         RegionData regionData = tableRegions[this.snapshotted++];
519         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
520         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
521         manifest.addRegion(regionData.tableDir, regionData.hri);
522         return regionData.files;
523       }
524 
525       public Path commit() throws IOException {
526         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
527         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
528         manifest.addTableDescriptor(htd);
529         manifest.consolidate();
530         SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
531         snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
532         return snapshotDir;
533       }
534     }
535 
536     public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
537       this.fs = fs;
538       this.conf = conf;
539       this.rootDir = rootDir;
540     }
541 
542     public SnapshotBuilder createSnapshotV1(final String snapshotName) throws IOException {
543       return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION);
544     }
545 
546     public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOException {
547       return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION);
548     }
549 
550     private SnapshotBuilder createSnapshot(final String snapshotName, final int version)
551         throws IOException {
552       HTableDescriptor htd = createHtd(snapshotName);
553       htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
554 
555       RegionData[] regions = createTable(htd, TEST_NUM_REGIONS);
556 
557       SnapshotDescription desc = SnapshotDescription.newBuilder()
558         .setTable(htd.getNameAsString())
559         .setName(snapshotName)
560         .setVersion(version)
561         .build();
562 
563       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
564       SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
565       return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
566     }
567 
568     public HTableDescriptor createHtd(final String tableName) {
569       HTableDescriptor htd = new HTableDescriptor(tableName);
570       htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
571       return htd;
572     }
573 
574     private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
575         throws IOException {
576       Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
577       new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
578 
579       assertTrue(nregions % 2 == 0);
580       RegionData[] regions = new RegionData[nregions];
581       for (int i = 0; i < regions.length; i += 2) {
582         byte[] startKey = Bytes.toBytes(0 + i * 2);
583         byte[] endKey = Bytes.toBytes(1 + i * 2);
584 
585         // First region, simple with one plain hfile.
586         HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
587         HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
588         regions[i] = new RegionData(tableDir, hri, 3);
589         for (int j = 0; j < regions[i].files.length; ++j) {
590           Path storeFile = createStoreFile(rfs.createTempName());
591           regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
592         }
593 
594         // Second region, used to test the split case.
595         // This region contains a reference to the hfile in the first region.
596         startKey = Bytes.toBytes(2 + i * 2);
597         endKey = Bytes.toBytes(3 + i * 2);
598         hri = new HRegionInfo(htd.getTableName());
599         rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
600         regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
601         for (int j = 0; j < regions[i].files.length; ++j) {
602           String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
603           Path refFile = createStoreFile(new Path(rootDir, refName));
604           regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
605         }
606       }
607       return regions;
608     }
609 
610     private Path createStoreFile(final Path storeFile)
611         throws IOException {
612       FSDataOutputStream out = fs.create(storeFile);
613       try {
614         out.write(Bytes.toBytes(storeFile.toString()));
615       } finally {
616         out.close();
617       }
618       return storeFile;
619     }
620   }
621 
622   // ==========================================================================
623   //  Table Helpers
624   // ==========================================================================
625   public static void waitForTableToBeOnline(final HBaseTestingUtility util,
626                                             final TableName tableName)
627       throws IOException, InterruptedException {
628     HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
629     List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
630     for (HRegion region : onlineRegions) {
631       region.waitForFlushesAndCompactions();
632     }
633     util.getHBaseAdmin().isTableAvailable(tableName);
634   }
635 
636   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
637       final byte[]... families) throws IOException, InterruptedException {
638     HTableDescriptor htd = new HTableDescriptor(tableName);
639     for (byte[] family: families) {
640       HColumnDescriptor hcd = new HColumnDescriptor(family);
641       htd.addFamily(hcd);
642     }
643     byte[][] splitKeys = new byte[KEYS.length-2][];
644     for (int i = 0; i < splitKeys.length; ++i) {
645       splitKeys[i] = new byte[] { KEYS[i+1] };
646     }
647     util.getHBaseAdmin().createTable(htd, splitKeys);
648     waitForTableToBeOnline(util, tableName);
649     assertEquals(KEYS.length-1, util.getHBaseAdmin().getTableRegions(tableName).size());
650   }
651 
652   public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
653       byte[]... families) throws IOException, InterruptedException {
654     loadData(util, new HTable(util.getConfiguration(), tableName), rows, families);
655   }
656 
657   public static void loadData(final HBaseTestingUtility util, final HTable table, int rows,
658       byte[]... families) throws IOException, InterruptedException {
659     table.setAutoFlush(false, true);
660 
661     // Ensure one row per region
662     assertTrue(rows >= KEYS.length);
663     for (byte k0: KEYS) {
664       byte[] k = new byte[] { k0 };
665       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
666       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
667       putData(table, families, key, value);
668       rows--;
669     }
670 
671     // Add other extra rows. more rows, more files
672     while (rows-- > 0) {
673       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
674       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
675       putData(table, families, key, value);
676     }
677     table.flushCommits();
678 
679     waitForTableToBeOnline(util, table.getName());
680   }
681 
682   private static void putData(final HTable table, final byte[][] families,
683       final byte[] key, final byte[] value) throws IOException {
684     byte[] q = Bytes.toBytes("q");
685     Put put = new Put(key);
686     put.setDurability(Durability.SKIP_WAL);
687     for (byte[] family: families) {
688       put.add(family, q, value);
689     }
690     table.put(put);
691   }
692 
693   public static void deleteAllSnapshots(final HBaseAdmin admin)
694       throws IOException {
695     // Delete all the snapshots
696     for (SnapshotDescription snapshot: admin.listSnapshots()) {
697       admin.deleteSnapshot(snapshot.getName());
698     }
699     SnapshotTestingUtils.assertNoSnapshots(admin);
700   }
701 
702   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
703       throws IOException {
704     // Ensure the archiver to be empty
705     MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
706     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
707     mfs.getFileSystem().delete(archiveDir, true);
708   }
709 
710   public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
711       long expectedRows) throws IOException {
712     HTable table = new HTable(util.getConfiguration(), tableName);
713     try {
714       assertEquals(expectedRows, util.countRows(table));
715     } finally {
716       table.close();
717     }
718   }
719 }