View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.Arrays;
26  import java.util.ArrayList;
27  import java.util.HashSet;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FSDataOutputStream;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.TableName;
45  import org.apache.hadoop.hbase.TableNotEnabledException;
46  import org.apache.hadoop.hbase.Waiter;
47  import org.apache.hadoop.hbase.client.Admin;
48  import org.apache.hadoop.hbase.client.BufferedMutator;
49  import org.apache.hadoop.hbase.client.Durability;
50  import org.apache.hadoop.hbase.client.HBaseAdmin;
51  import org.apache.hadoop.hbase.client.HTable;
52  import org.apache.hadoop.hbase.client.Put;
53  import org.apache.hadoop.hbase.client.Table;
54  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
55  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
56  import org.apache.hadoop.hbase.io.HFileLink;
57  import org.apache.hadoop.hbase.master.HMaster;
58  import org.apache.hadoop.hbase.master.MasterFileSystem;
59  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
61  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
62  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
63  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
64  import org.apache.hadoop.hbase.regionserver.HRegion;
65  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
66  import org.apache.hadoop.hbase.regionserver.HRegionServer;
67  import org.apache.hadoop.hbase.util.Bytes;
68  import org.apache.hadoop.hbase.util.FSTableDescriptors;
69  import org.apache.hadoop.hbase.util.FSVisitor;
70  import org.apache.hadoop.hbase.util.FSUtils;
71  import org.apache.hadoop.hbase.util.MD5Hash;
72  import org.junit.Assert;
73  
74  import com.google.protobuf.ServiceException;
75  
76  /**
77   * Utilities class for snapshots
78   */
79  public class SnapshotTestingUtils {
80  
81    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
82    private static byte[] KEYS = Bytes.toBytes("0123456789");
83  
84    /**
85     * Assert that we don't have any snapshots lists
86     *
87     * @throws IOException
88     *           if the admin operation fails
89     */
90    public static void assertNoSnapshots(Admin admin) throws IOException {
91      assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
92          .size());
93    }
94  
95    /**
96     * Make sure that there is only one snapshot returned from the master and its
97     * name and table match the passed in parameters.
98     */
99    public static List<SnapshotDescription> assertExistsMatchingSnapshot(
100       Admin admin, String snapshotName, TableName tableName)
101       throws IOException {
102     // list the snapshot
103     List<SnapshotDescription> snapshots = admin.listSnapshots();
104 
105     List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
106     for (SnapshotDescription sd : snapshots) {
107       if (snapshotName.equals(sd.getName()) &&
108           tableName.equals(TableName.valueOf(sd.getTable()))) {
109         returnedSnapshots.add(sd);
110       }
111     }
112 
113     Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
114     return returnedSnapshots;
115   }
116 
117   /**
118    * Make sure that there is only one snapshot returned from the master
119    */
120   public static void assertOneSnapshotThatMatches(Admin admin,
121       SnapshotDescription snapshot) throws IOException {
122     assertOneSnapshotThatMatches(admin, snapshot.getName(),
123         TableName.valueOf(snapshot.getTable()));
124   }
125 
126   /**
127    * Make sure that there is only one snapshot returned from the master and its
128    * name and table match the passed in parameters.
129    */
130   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
131       Admin admin, String snapshotName, TableName tableName)
132       throws IOException {
133     // list the snapshot
134     List<SnapshotDescription> snapshots = admin.listSnapshots();
135 
136     assertEquals("Should only have 1 snapshot", 1, snapshots.size());
137     assertEquals(snapshotName, snapshots.get(0).getName());
138     assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
139 
140     return snapshots;
141   }
142 
143   /**
144    * Make sure that there is only one snapshot returned from the master and its
145    * name and table match the passed in parameters.
146    */
147   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
148       Admin admin, byte[] snapshot, TableName tableName) throws IOException {
149     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
150         tableName);
151   }
152 
153   /**
154    * Confirm that the snapshot contains references to all the files that should
155    * be in the snapshot.
156    */
157   public static void confirmSnapshotValid(
158       SnapshotDescription snapshotDescriptor, TableName tableName,
159       byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
160       throws IOException {
161     ArrayList nonEmptyTestFamilies = new ArrayList(1);
162     nonEmptyTestFamilies.add(testFamily);
163     confirmSnapshotValid(snapshotDescriptor, tableName,
164       nonEmptyTestFamilies, null, rootDir, admin, fs);
165   }
166 
167   /**
168    * Confirm that the snapshot has no references files but only metadata.
169    */
170   public static void confirmEmptySnapshotValid(
171       SnapshotDescription snapshotDescriptor, TableName tableName,
172       byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
173       throws IOException {
174     ArrayList emptyTestFamilies = new ArrayList(1);
175     emptyTestFamilies.add(testFamily);
176     confirmSnapshotValid(snapshotDescriptor, tableName,
177       null, emptyTestFamilies, rootDir, admin, fs);
178   }
179 
180   /**
181    * Confirm that the snapshot contains references to all the files that should
182    * be in the snapshot. This method also perform some redundant check like
183    * the existence of the snapshotinfo or the regioninfo which are done always
184    * by the MasterSnapshotVerifier, at the end of the snapshot operation.
185    */
186   public static void confirmSnapshotValid(
187       SnapshotDescription snapshotDescriptor, TableName tableName,
188       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
189       Path rootDir, Admin admin, FileSystem fs) throws IOException {
190     final Configuration conf = admin.getConfiguration();
191 
192     // check snapshot dir
193     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
194         snapshotDescriptor, rootDir);
195     assertTrue(fs.exists(snapshotDir));
196 
197     SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
198 
199     // Extract regions and families with store files
200     final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
201 
202     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
203     Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
204     for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
205       SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
206           new SnapshotReferenceUtil.StoreFileVisitor() {
207         @Override
208         public void storeFile(final HRegionInfo regionInfo, final String family,
209               final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
210           snapshotFamilies.add(Bytes.toBytes(family));
211         }
212       });
213     }
214 
215     // Verify that there are store files in the specified families
216     if (nonEmptyTestFamilies != null) {
217       for (final byte[] familyName: nonEmptyTestFamilies) {
218         assertTrue(snapshotFamilies.contains(familyName));
219       }
220     }
221 
222     // Verify that there are no store files in the specified families
223     if (emptyTestFamilies != null) {
224       for (final byte[] familyName: emptyTestFamilies) {
225         assertFalse(snapshotFamilies.contains(familyName));
226       }
227     }
228 
229     // check the region snapshot for all the regions
230     List<HRegionInfo> regions = admin.getTableRegions(tableName);
231     // remove the non-default regions
232     RegionReplicaUtil.removeNonDefaultRegions(regions);
233     assertEquals(regions.size(), regionManifests.size());
234 
235     // Verify Regions (redundant check, see MasterSnapshotVerifier)
236     for (HRegionInfo info : regions) {
237       String regionName = info.getEncodedName();
238       assertTrue(regionManifests.containsKey(regionName));
239     }
240   }
241 
242   /**
243    * Helper method for testing async snapshot operations. Just waits for the
244    * given snapshot to complete on the server by repeatedly checking the master.
245    *
246    * @param master: the master running the snapshot
247    * @param snapshot: the snapshot to check
248    * @param sleep: amount to sleep between checks to see if the snapshot is done
249    * @throws ServiceException if the snapshot fails
250    */
251   public static void waitForSnapshotToComplete(HMaster master,
252       SnapshotDescription snapshot, long sleep) throws ServiceException {
253     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
254         .setSnapshot(snapshot).build();
255     IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
256         .buildPartial();
257     while (!done.getDone()) {
258       done = master.getMasterRpcServices().isSnapshotDone(null, request);
259       try {
260         Thread.sleep(sleep);
261       } catch (InterruptedException e) {
262         throw new ServiceException(e);
263       }
264     }
265   }
266 
267   /*
268    * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
269    * except for the last CorruptedSnapshotException
270    */
271   public static void snapshot(Admin admin,
272       final String snapshotName, final String tableName,
273       SnapshotDescription.Type type, int numTries) throws IOException {
274     int tries = 0;
275     CorruptedSnapshotException lastEx = null;
276     while (tries++ < numTries) {
277       try {
278         admin.snapshot(snapshotName, TableName.valueOf(tableName), type);
279         return;
280       } catch (CorruptedSnapshotException cse) {
281         LOG.warn("Got CorruptedSnapshotException", cse);
282         lastEx = cse;
283       }
284     }
285     throw lastEx;
286   }
287 
288   public static void cleanupSnapshot(Admin admin, byte[] tableName)
289       throws IOException {
290     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
291   }
292 
293   public static void cleanupSnapshot(Admin admin, String snapshotName)
294       throws IOException {
295     // delete the taken snapshot
296     admin.deleteSnapshot(snapshotName);
297     assertNoSnapshots(admin);
298   }
299 
300   /**
301    * Expect the snapshot to throw an error when checking if the snapshot is
302    * complete
303    *
304    * @param master master to check
305    * @param snapshot the {@link SnapshotDescription} request to pass to the master
306    * @param clazz expected exception from the master
307    */
308   public static void expectSnapshotDoneException(HMaster master,
309       IsSnapshotDoneRequest snapshot,
310       Class<? extends HBaseSnapshotException> clazz) {
311     try {
312       master.getMasterRpcServices().isSnapshotDone(null, snapshot);
313       Assert.fail("didn't fail to lookup a snapshot");
314     } catch (ServiceException se) {
315       try {
316         throw ProtobufUtil.getRemoteException(se);
317       } catch (HBaseSnapshotException e) {
318         assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
319       } catch (Throwable t) {
320         Assert.fail("Threw an unexpected exception:" + t);
321       }
322     }
323   }
324 
325   /**
326    * List all the HFiles in the given table
327    *
328    * @param fs: FileSystem where the table lives
329    * @param tableDir directory of the table
330    * @return array of the current HFiles in the table (could be a zero-length array)
331    * @throws IOException on unexecpted error reading the FS
332    */
333   public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
334       throws IOException {
335     final ArrayList<Path> hfiles = new ArrayList<Path>();
336     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
337       @Override
338       public void storeFile(final String region, final String family, final String hfileName)
339           throws IOException {
340         hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
341       }
342     });
343     return hfiles.toArray(new Path[hfiles.size()]);
344   }
345 
346   public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
347       throws IOException {
348     Path[] files = listHFiles(fs, tableDir);
349     String[] names = new String[files.length];
350     for (int i = 0; i < files.length; ++i) {
351       names[i] = files[i].getName();
352     }
353     Arrays.sort(names);
354     return names;
355   }
356 
357   /**
358    * Take a snapshot of the specified table and verify that the given family is
359    * not empty. Note that this will leave the table disabled
360    * in the case of an offline snapshot.
361    */
362   public static void createSnapshotAndValidate(Admin admin,
363       TableName tableName, String familyName, String snapshotNameString,
364       Path rootDir, FileSystem fs, boolean onlineSnapshot)
365       throws Exception {
366     ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
367     nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
368     createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
369                               snapshotNameString, rootDir, fs, onlineSnapshot);
370   }
371 
372   /**
373    * Take a snapshot of the specified table and verify the given families.
374    * Note that this will leave the table disabled in the case of an offline snapshot.
375    */
376   public static void createSnapshotAndValidate(Admin admin,
377       TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
378       String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
379         throws Exception {
380     if (!onlineSnapshot) {
381       try {
382         admin.disableTable(tableName);
383       } catch (TableNotEnabledException tne) {
384         LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
385             "already disabled.");
386       }
387     }
388     admin.snapshot(snapshotNameString, tableName);
389 
390     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
391       snapshotNameString, tableName);
392     if (snapshots == null || snapshots.size() != 1) {
393       Assert.fail("Incorrect number of snapshots for table " + tableName);
394     }
395 
396     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
397       emptyFamilyNames, rootDir, admin, fs);
398   }
399 
400   /**
401    * Corrupt the specified snapshot by deleting some files.
402    *
403    * @param util {@link HBaseTestingUtility}
404    * @param snapshotName name of the snapshot to corrupt
405    * @return array of the corrupted HFiles
406    * @throws IOException on unexecpted error reading the FS
407    */
408   public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
409       throws IOException {
410     final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
411     final FileSystem fs = mfs.getFileSystem();
412 
413     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
414                                                                         mfs.getRootDir());
415     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
416     final TableName table = TableName.valueOf(snapshotDesc.getTable());
417 
418     final ArrayList corruptedFiles = new ArrayList();
419     final Configuration conf = util.getConfiguration();
420     SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
421         new SnapshotReferenceUtil.StoreFileVisitor() {
422       @Override
423       public void storeFile(final HRegionInfo regionInfo, final String family,
424             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
425         String region = regionInfo.getEncodedName();
426         String hfile = storeFile.getName();
427         HFileLink link = HFileLink.create(conf, table, region, family, hfile);
428         if (corruptedFiles.size() % 2 == 0) {
429           fs.delete(link.getAvailablePath(fs), true);
430           corruptedFiles.add(hfile);
431         }
432       }
433     });
434 
435     assertTrue(corruptedFiles.size() > 0);
436     return corruptedFiles;
437   }
438 
439   // ==========================================================================
440   //  Snapshot Mock
441   // ==========================================================================
442   public static class SnapshotMock {
443     private final static String TEST_FAMILY = "cf";
444     public final static int TEST_NUM_REGIONS = 4;
445 
446     private final Configuration conf;
447     private final FileSystem fs;
448     private final Path rootDir;
449 
450     static class RegionData {
451       public HRegionInfo hri;
452       public Path tableDir;
453       public Path[] files;
454 
455       public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
456         this.tableDir = tableDir;
457         this.hri = hri;
458         this.files = new Path[nfiles];
459       }
460     }
461 
462     public static class SnapshotBuilder {
463       private final RegionData[] tableRegions;
464       private final SnapshotDescription desc;
465       private final HTableDescriptor htd;
466       private final Configuration conf;
467       private final FileSystem fs;
468       private final Path rootDir;
469       private Path snapshotDir;
470       private int snapshotted = 0;
471 
472       public SnapshotBuilder(final Configuration conf, final FileSystem fs,
473           final Path rootDir, final HTableDescriptor htd,
474           final SnapshotDescription desc, final RegionData[] tableRegions)
475           throws IOException {
476         this.fs = fs;
477         this.conf = conf;
478         this.rootDir = rootDir;
479         this.htd = htd;
480         this.desc = desc;
481         this.tableRegions = tableRegions;
482         this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
483         new FSTableDescriptors(conf)
484           .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
485       }
486 
487       public HTableDescriptor getTableDescriptor() {
488         return this.htd;
489       }
490 
491       public SnapshotDescription getSnapshotDescription() {
492         return this.desc;
493       }
494 
495       public Path getSnapshotsDir() {
496         return this.snapshotDir;
497       }
498 
499       public Path[] addRegion() throws IOException {
500         return addRegion(desc);
501       }
502 
503       public Path[] addRegionV1() throws IOException {
504         return addRegion(desc.toBuilder()
505                           .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
506                           .build());
507       }
508 
509       public Path[] addRegionV2() throws IOException {
510         return addRegion(desc.toBuilder()
511                           .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
512                           .build());
513       }
514 
515       private Path[] addRegion(final SnapshotDescription desc) throws IOException {
516         if (this.snapshotted == tableRegions.length) {
517           throw new UnsupportedOperationException("No more regions in the table");
518         }
519 
520         RegionData regionData = tableRegions[this.snapshotted++];
521         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
522         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
523         manifest.addRegion(regionData.tableDir, regionData.hri);
524         return regionData.files;
525       }
526 
527       public Path commit() throws IOException {
528         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
529         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
530         manifest.addTableDescriptor(htd);
531         manifest.consolidate();
532         SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
533         snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
534         return snapshotDir;
535       }
536     }
537 
538     public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
539       this.fs = fs;
540       this.conf = conf;
541       this.rootDir = rootDir;
542     }
543 
544     public SnapshotBuilder createSnapshotV1(final String snapshotName) throws IOException {
545       return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION);
546     }
547 
548     public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOException {
549       return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION);
550     }
551 
552     private SnapshotBuilder createSnapshot(final String snapshotName, final int version)
553         throws IOException {
554       HTableDescriptor htd = createHtd(snapshotName);
555 
556       RegionData[] regions = createTable(htd, TEST_NUM_REGIONS);
557 
558       SnapshotDescription desc = SnapshotDescription.newBuilder()
559         .setTable(htd.getNameAsString())
560         .setName(snapshotName)
561         .setVersion(version)
562         .build();
563 
564       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
565       SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
566       return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
567     }
568 
569     public HTableDescriptor createHtd(final String tableName) {
570       HTableDescriptor htd = new HTableDescriptor(tableName);
571       htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
572       return htd;
573     }
574 
575     private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
576         throws IOException {
577       Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
578       new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
579 
580       assertTrue(nregions % 2 == 0);
581       RegionData[] regions = new RegionData[nregions];
582       for (int i = 0; i < regions.length; i += 2) {
583         byte[] startKey = Bytes.toBytes(0 + i * 2);
584         byte[] endKey = Bytes.toBytes(1 + i * 2);
585 
586         // First region, simple with one plain hfile.
587         HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
588         HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
589         regions[i] = new RegionData(tableDir, hri, 3);
590         for (int j = 0; j < regions[i].files.length; ++j) {
591           Path storeFile = createStoreFile(rfs.createTempName());
592           regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
593         }
594 
595         // Second region, used to test the split case.
596         // This region contains a reference to the hfile in the first region.
597         startKey = Bytes.toBytes(2 + i * 2);
598         endKey = Bytes.toBytes(3 + i * 2);
599         hri = new HRegionInfo(htd.getTableName());
600         rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
601         regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
602         for (int j = 0; j < regions[i].files.length; ++j) {
603           String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
604           Path refFile = createStoreFile(new Path(rootDir, refName));
605           regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
606         }
607       }
608       return regions;
609     }
610 
611     private Path createStoreFile(final Path storeFile)
612         throws IOException {
613       FSDataOutputStream out = fs.create(storeFile);
614       try {
615         out.write(Bytes.toBytes(storeFile.toString()));
616       } finally {
617         out.close();
618       }
619       return storeFile;
620     }
621   }
622 
623   // ==========================================================================
624   //  Table Helpers
625   // ==========================================================================
626   public static void waitForTableToBeOnline(final HBaseTestingUtility util,
627                                             final TableName tableName)
628       throws IOException, InterruptedException {
629     HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
630     List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
631     for (HRegion region : onlineRegions) {
632       region.waitForFlushesAndCompactions();
633     }
634     // Wait up to 60 seconds for a table to be available.
635     final HBaseAdmin hBaseAdmin = util.getHBaseAdmin();
636     util.waitFor(60000, new Waiter.Predicate<IOException>() {
637       @Override
638       public boolean evaluate() throws IOException {
639         return hBaseAdmin.isTableAvailable(tableName);
640       }
641     });
642   }
643 
644   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
645       int regionReplication, final byte[]... families) throws IOException, InterruptedException {
646     HTableDescriptor htd = new HTableDescriptor(tableName);
647     htd.setRegionReplication(regionReplication);
648     for (byte[] family : families) {
649       HColumnDescriptor hcd = new HColumnDescriptor(family);
650       htd.addFamily(hcd);
651     }
652     byte[][] splitKeys = getSplitKeys();
653     util.createTable(htd, splitKeys);
654     assertEquals((splitKeys.length + 1) * regionReplication,
655         util.getHBaseAdmin().getTableRegions(tableName).size());
656   }
657 
658   public static byte[][] getSplitKeys() {
659     byte[][] splitKeys = new byte[KEYS.length-2][];
660     for (int i = 0; i < splitKeys.length; ++i) {
661       splitKeys[i] = new byte[] { KEYS[i+1] };
662     }
663     return splitKeys;
664   }
665 
666   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
667       final byte[]... families) throws IOException, InterruptedException {
668     createTable(util, tableName, 1, families);
669   }
670 
671   public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
672       byte[]... families) throws IOException, InterruptedException {
673     try (BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName)) {
674       loadData(util, mutator, rows, families);
675     }
676   }
677 
678   public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
679       byte[]... families) throws IOException, InterruptedException {
680     // Ensure one row per region
681     assertTrue(rows >= KEYS.length);
682     for (byte k0: KEYS) {
683       byte[] k = new byte[] { k0 };
684       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
685       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
686       final byte[][] families1 = families;
687       final byte[] key1 = key;
688       final byte[] value1 = value;
689       mutator.mutate(createPut(families1, key1, value1));
690       rows--;
691     }
692 
693     // Add other extra rows. more rows, more files
694     while (rows-- > 0) {
695       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
696       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
697       final byte[][] families1 = families;
698       final byte[] key1 = key;
699       final byte[] value1 = value;
700       mutator.mutate(createPut(families1, key1, value1));
701     }
702     mutator.flush();
703 
704     waitForTableToBeOnline(util, mutator.getName());
705   }
706 
707   private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
708     byte[] q = Bytes.toBytes("q");
709     Put put = new Put(key);
710     put.setDurability(Durability.SKIP_WAL);
711     for (byte[] family: families) {
712       put.add(family, q, value);
713     }
714     return put;
715   }
716 
717   public static void deleteAllSnapshots(final Admin admin)
718       throws IOException {
719     // Delete all the snapshots
720     for (SnapshotDescription snapshot: admin.listSnapshots()) {
721       admin.deleteSnapshot(snapshot.getName());
722     }
723     SnapshotTestingUtils.assertNoSnapshots(admin);
724   }
725 
726   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
727       throws IOException {
728     // Ensure the archiver to be empty
729     MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
730     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
731     mfs.getFileSystem().delete(archiveDir, true);
732   }
733 
734   public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
735       long expectedRows) throws IOException {
736     Table table = new HTable(util.getConfiguration(), tableName);
737     try {
738       assertEquals(expectedRows, util.countRows(table));
739     } finally {
740       table.close();
741     }
742   }
743 
744   public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
745       int regionReplication) throws IOException {
746     List<HRegionInfo> regions = admin.getTableRegions(tableName);
747     HashSet<HRegionInfo> set = new HashSet<HRegionInfo>();
748     for (HRegionInfo hri : regions) {
749       set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
750       for (int i = 0; i < regionReplication; i++) {
751         HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
752         if (!regions.contains(replica)) {
753           Assert.fail(replica + " is not contained in the list of online regions");
754         }
755       }
756     }
757     assert(set.size() == getSplitKeys().length + 1);
758   }
759 }