View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collection;
27  import java.util.List;
28  import java.util.Set;
29  import java.util.TreeSet;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.fs.PathFilter;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.TableNotEnabledException;
44  import org.apache.hadoop.hbase.client.Durability;
45  import org.apache.hadoop.hbase.client.HBaseAdmin;
46  import org.apache.hadoop.hbase.client.HTable;
47  import org.apache.hadoop.hbase.client.Put;
48  import org.apache.hadoop.hbase.master.HMaster;
49  import org.apache.hadoop.hbase.master.MasterFileSystem;
50  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
51  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
52  import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsSnapshotDoneRequest;
53  import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsSnapshotDoneResponse;
54  import org.apache.hadoop.hbase.regionserver.HRegion;
55  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
56  import org.apache.hadoop.hbase.regionserver.HRegionServer;
57  import org.apache.hadoop.hbase.util.Bytes;
58  import org.apache.hadoop.hbase.util.FSTableDescriptors;
59  import org.apache.hadoop.hbase.util.FSUtils;
60  import org.apache.hadoop.hbase.util.MD5Hash;
61  import org.junit.Assert;
62  
63  import com.google.protobuf.ServiceException;
64  
65  /**
66   * Utilities class for snapshots
67   */
68  public class SnapshotTestingUtils {
69  
70    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
71  
72    /**
73     * Assert that we don't have any snapshots lists
74     *
75     * @throws IOException
76     *           if the admin operation fails
77     */
78    public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
79      assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
80          .size());
81    }
82  
83    /**
84     * Make sure that there is only one snapshot returned from the master and its
85     * name and table match the passed in parameters.
86     */
87    public static List<SnapshotDescription> assertExistsMatchingSnapshot(
88        HBaseAdmin admin, String snapshotName, TableName tableName)
89        throws IOException {
90      // list the snapshot
91      List<SnapshotDescription> snapshots = admin.listSnapshots();
92  
93      List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
94      for (SnapshotDescription sd : snapshots) {
95        if (snapshotName.equals(sd.getName()) &&
96            tableName.equals(TableName.valueOf(sd.getTable()))) {
97          returnedSnapshots.add(sd);
98        }
99      }
100 
101     Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
102     return returnedSnapshots;
103   }
104 
105   /**
106    * Make sure that there is only one snapshot returned from the master
107    */
108   public static void assertOneSnapshotThatMatches(HBaseAdmin admin,
109       SnapshotDescription snapshot) throws IOException {
110     assertOneSnapshotThatMatches(admin, snapshot.getName(),
111         TableName.valueOf(snapshot.getTable()));
112   }
113 
114   /**
115    * Make sure that there is only one snapshot returned from the master and its
116    * name and table match the passed in parameters.
117    */
118   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
119       HBaseAdmin admin, String snapshotName, TableName tableName)
120       throws IOException {
121     // list the snapshot
122     List<SnapshotDescription> snapshots = admin.listSnapshots();
123 
124     assertEquals("Should only have 1 snapshot", 1, snapshots.size());
125     assertEquals(snapshotName, snapshots.get(0).getName());
126     assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
127 
128     return snapshots;
129   }
130 
131   /**
132    * Make sure that there is only one snapshot returned from the master and its
133    * name and table match the passed in parameters.
134    */
135   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
136       HBaseAdmin admin, byte[] snapshot, TableName tableName) throws IOException {
137     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
138         tableName);
139   }
140 
141   /**
142    * Multi-family version of the confirmSnapshotValid function
143    */
144   public static void confirmSnapshotValid(
145       SnapshotDescription snapshotDescriptor, TableName tableName,
146       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
147       Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs,
148       Path logsDir, Set<String> snapshotServers) throws IOException {
149     if (nonEmptyTestFamilies != null) {
150       for (byte[] testFamily : nonEmptyTestFamilies) {
151         confirmSnapshotValid(snapshotDescriptor, tableName, testFamily,
152             rootDir, admin, fs, requireLogs, logsDir, false, null);
153       }
154     }
155 
156     if (emptyTestFamilies != null) {
157       for (byte[] testFamily : emptyTestFamilies) {
158         confirmSnapshotValid(snapshotDescriptor, tableName, testFamily,
159             rootDir, admin, fs, requireLogs, logsDir, true, null);
160       }
161     }
162   }
163 
164   /**
165    * Confirm that the snapshot contains references to all the files that should
166    * be in the snapshot.
167    */
168   public static void confirmSnapshotValid(
169       SnapshotDescription snapshotDescriptor, TableName tableName,
170       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
171       boolean requireLogs, Path logsDir, Set<String> snapshotServers)
172       throws IOException {
173     confirmSnapshotValid(snapshotDescriptor, tableName, testFamily, rootDir,
174         admin, fs, requireLogs, logsDir, false, snapshotServers);
175   }
176 
177   /**
178    * Confirm that the snapshot contains references to all the files that should
179    * be in the snapshot.
180    */
181   public static void confirmSnapshotValid(
182       SnapshotDescription snapshotDescriptor, TableName tableName,
183       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
184       boolean requireLogs, Path logsDir, boolean familyEmpty,
185       Set<String> snapshotServers) throws IOException {
186     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
187         snapshotDescriptor, rootDir);
188     assertTrue(fs.exists(snapshotDir));
189     Path snapshotinfo = new Path(snapshotDir,
190         SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
191     assertTrue(fs.exists(snapshotinfo));
192     // check the logs dir
193     if (requireLogs) {
194       TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir,
195           snapshotServers, snapshotDescriptor, new Path(snapshotDir,
196               HConstants.HREGION_LOGDIR_NAME));
197     }
198     // check the table info
199     HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir,
200         tableName);
201     HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs,
202         snapshotDir);
203     assertEquals(desc, snapshotDesc);
204 
205     // check the region snapshot for all the regions
206     List<HRegionInfo> regions = admin.getTableRegions(tableName);
207     for (HRegionInfo info : regions) {
208       String regionName = info.getEncodedName();
209       Path regionDir = new Path(snapshotDir, regionName);
210       HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
211       assertEquals(info, snapshotRegionInfo);
212 
213       // check to make sure we have the family
214       if (!familyEmpty) {
215         Path familyDir = new Path(regionDir, Bytes.toString(testFamily));
216         assertTrue("Expected to find: " + familyDir + ", but it doesn't exist",
217             fs.exists(familyDir));
218         // make sure we have some files references
219         assertTrue(fs.listStatus(familyDir).length > 0);
220       }
221     }
222   }
223 
224   /**
225    * Helper method for testing async snapshot operations. Just waits for the
226    * given snapshot to complete on the server by repeatedly checking the master.
227    *
228    * @param master: the master running the snapshot
229    * @param snapshot: the snapshot to check
230    * @param sleep: amount to sleep between checks to see if the snapshot is done
231    * @throws ServiceException if the snapshot fails
232    */
233   public static void waitForSnapshotToComplete(HMaster master,
234       SnapshotDescription snapshot, long sleep) throws ServiceException {
235     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
236         .setSnapshot(snapshot).build();
237     IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
238         .buildPartial();
239     while (!done.getDone()) {
240       done = master.isSnapshotDone(null, request);
241       try {
242         Thread.sleep(sleep);
243       } catch (InterruptedException e) {
244         throw new ServiceException(e);
245       }
246     }
247   }
248 
249   public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName)
250       throws IOException {
251     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
252   }
253 
254   public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName)
255       throws IOException {
256     // delete the taken snapshot
257     admin.deleteSnapshot(snapshotName);
258     assertNoSnapshots(admin);
259   }
260 
261   /**
262    * Expect the snapshot to throw an error when checking if the snapshot is
263    * complete
264    *
265    * @param master master to check
266    * @param snapshot the {@link SnapshotDescription} request to pass to the master
267    * @param clazz expected exception from the master
268    */
269   public static void expectSnapshotDoneException(HMaster master,
270       IsSnapshotDoneRequest snapshot,
271       Class<? extends HBaseSnapshotException> clazz) {
272     try {
273       master.isSnapshotDone(null, snapshot);
274       Assert.fail("didn't fail to lookup a snapshot");
275     } catch (ServiceException se) {
276       try {
277         throw ProtobufUtil.getRemoteException(se);
278       } catch (HBaseSnapshotException e) {
279         assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
280       } catch (Throwable t) {
281         Assert.fail("Threw an unexpected exception:" + t);
282       }
283     }
284   }
285 
286   /**
287    * List all the HFiles in the given table
288    *
289    * @param fs: FileSystem where the table lives
290    * @param tableDir directory of the table
291    * @return array of the current HFiles in the table (could be a zero-length array)
292    * @throws IOException on unexecpted error reading the FS
293    */
294   public static FileStatus[] listHFiles(final FileSystem fs, Path tableDir)
295       throws IOException {
296     // setup the filters we will need based on the filesystem
297     PathFilter regionFilter = new FSUtils.RegionDirFilter(fs);
298     PathFilter familyFilter = new FSUtils.FamilyDirFilter(fs);
299     final PathFilter fileFilter = new PathFilter() {
300       @Override
301       public boolean accept(Path file) {
302         try {
303           return fs.isFile(file);
304         } catch (IOException e) {
305           return false;
306         }
307       }
308     };
309 
310     FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, regionFilter);
311     // if no regions, then we are done
312     if (regionDirs == null || regionDirs.length == 0)
313       return new FileStatus[0];
314 
315     // go through each of the regions, and add al the hfiles under each family
316     List<FileStatus> regionFiles = new ArrayList<FileStatus>(regionDirs.length);
317     for (FileStatus regionDir : regionDirs) {
318       FileStatus[] fams = FSUtils.listStatus(fs, regionDir.getPath(),
319           familyFilter);
320       // if no families, then we are done again
321       if (fams == null || fams.length == 0)
322         continue;
323       // add all the hfiles under the family
324       regionFiles.addAll(SnapshotTestingUtils.getHFilesInRegion(fams, fs,
325           fileFilter));
326     }
327     FileStatus[] files = new FileStatus[regionFiles.size()];
328     regionFiles.toArray(files);
329     return files;
330   }
331 
332   /**
333    * Get all the hfiles in the region, under the passed set of families
334    *
335    * @param families: all the family directories under the region
336    * @param fs: filesystem where the families live
337    * @param fileFilter: filter to only include files
338    * @return collection of all the hfiles under all the passed in families (non-null)
339    * @throws IOException on unexecpted error reading the FS
340    */
341   public static Collection<FileStatus> getHFilesInRegion(FileStatus[] families,
342       FileSystem fs, PathFilter fileFilter) throws IOException {
343     Set<FileStatus> files = new TreeSet<FileStatus>();
344     for (FileStatus family : families) {
345       // get all the hfiles in the family
346       FileStatus[] hfiles = FSUtils
347           .listStatus(fs, family.getPath(), fileFilter);
348       // if no hfiles, then we are done with this family
349       if (hfiles == null || hfiles.length == 0)
350         continue;
351       files.addAll(Arrays.asList(hfiles));
352     }
353     return files;
354   }
355 
356   /**
357    * Take an offline snapshot of the specified table and verify if the given
358    * family is empty. Note that this will leave the table disabled
359    * in the case of an offline snapshot.
360    */
361   public static void createOfflineSnapshotAndValidate(HBaseAdmin admin,
362       TableName tableName, String familyName, String snapshotNameString,
363       Path rootDir, FileSystem fs, boolean familyEmpty) throws Exception {
364 
365     createSnapshotAndValidate(admin, tableName, familyName,
366         snapshotNameString, rootDir, fs, familyEmpty, false);
367   }
368 
369   /**
370    * Take a snapshot of the specified table and verify if the given family is
371    * empty. Note that this will leave the table disabled
372    * in the case of an offline snapshot.
373    */
374   public static void createSnapshotAndValidate(HBaseAdmin admin,
375       TableName tableName, String familyName, String snapshotNameString,
376       Path rootDir, FileSystem fs, boolean familyEmpty, boolean onlineSnapshot)
377       throws Exception {
378 
379     if (!onlineSnapshot) {
380       try {
381         admin.disableTable(tableName);
382       } catch (TableNotEnabledException tne) {
383         LOG.info("In attempting to disable " + tableName
384             + " it turns out that this table is already disabled.");
385       }
386     }
387 
388     admin.snapshot(snapshotNameString, tableName);
389 
390     List<SnapshotDescription> snapshots = SnapshotTestingUtils
391         .assertExistsMatchingSnapshot(admin, snapshotNameString,
392             tableName);
393 
394     if (snapshots == null || snapshots.size() != 1) {
395       Assert.fail("Incorrect number of snapshots for table "
396           + tableName);
397     }
398 
399     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName,
400         Bytes.toBytes(familyName), rootDir, admin, fs, false, new Path(rootDir,
401             HConstants.HREGION_LOGDIR_NAME), familyEmpty, null);
402   }
403   public static void createSnapshotAndValidate(HBaseAdmin admin,
404       TableName tableName, String familyName, String snapshotNameString,
405       Path rootDir, FileSystem fs) throws Exception {
406     createSnapshotAndValidate(admin, tableName, familyName,
407         snapshotNameString, rootDir, fs, false, false);
408   }
409 
410   /**
411    * This will create a snapshot. Note that this has the side effect
412    * of leaving the input table disabled if the offline snapshot
413    * option is chosen.
414    *
415    */
416   public static void createSnapshotAndValidate(HBaseAdmin admin,
417       TableName tableName, String familyName, String snapshotNameString,
418       Path rootDir, FileSystem fs, boolean online) throws Exception {
419     createSnapshotAndValidate(admin, tableName, familyName,
420         snapshotNameString, rootDir, fs, false, online);
421   }
422 
423   public static void createSnapshotAndValidate(HBaseAdmin admin,
424       TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
425       String snapshotNameString, Path rootDir, FileSystem fs) throws Exception {
426 
427     try {
428       admin.disableTable(tableName);
429     } catch (TableNotEnabledException tne) {
430       LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
431           "already disabled.");
432     }
433     admin.snapshot(snapshotNameString, tableName);
434 
435     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
436       snapshotNameString, tableName);
437 
438     // Create test-timestamp-clone
439     if (snapshots == null || snapshots.size() != 1) {
440       Assert.fail("Incorrect number of snapshots for table " + tableName);
441     }
442 
443     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames, emptyFamilyNames,
444       rootDir, admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
445   }
446 
447   // ==========================================================================
448   //  Table Helpers
449   // ==========================================================================
450   public static void waitForTableToBeOnline(final HBaseTestingUtility util,
451                                             final TableName tableName)
452       throws IOException, InterruptedException {
453     HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
454     List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
455     for (HRegion region : onlineRegions) {
456       region.waitForFlushesAndCompactions();
457     }
458     util.getHBaseAdmin().isTableAvailable(tableName);
459   }
460 
461   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
462       final byte[]... families) throws IOException, InterruptedException {
463     HTableDescriptor htd = new HTableDescriptor(tableName);
464     for (byte[] family: families) {
465       HColumnDescriptor hcd = new HColumnDescriptor(family);
466       htd.addFamily(hcd);
467     }
468     byte[][] splitKeys = new byte[14][];
469     byte[] hex = Bytes.toBytes("123456789abcde");
470     for (int i = 0; i < splitKeys.length; ++i) {
471       splitKeys[i] = new byte[] { hex[i] };
472     }
473     util.getHBaseAdmin().createTable(htd, splitKeys);
474     waitForTableToBeOnline(util, tableName);
475     assertEquals(15, util.getHBaseAdmin().getTableRegions(tableName).size());
476   }
477 
478   public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
479       byte[]... families) throws IOException, InterruptedException {
480     loadData(util, new HTable(util.getConfiguration(), tableName), rows, families);
481   }
482 
483   public static void loadData(final HBaseTestingUtility util, final HTable table, int rows,
484       byte[]... families) throws IOException, InterruptedException {
485     table.setAutoFlush(false);
486 
487     // Ensure one row per region
488     assertTrue(rows >= 16);
489     for (byte k0: Bytes.toBytes("0123456789abcdef")) {
490       byte[] k = new byte[] { k0 };
491       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
492       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
493       putData(table, families, key, value);
494       rows--;
495     }
496 
497     // Add other extra rows. more rows, more files
498     while (rows-- > 0) {
499       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
500       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
501       putData(table, families, key, value);
502     }
503     table.flushCommits();
504 
505     waitForTableToBeOnline(util, table.getName());
506   }
507 
508   private static void putData(final HTable table, final byte[][] families,
509       final byte[] key, final byte[] value) throws IOException {
510     byte[] q = Bytes.toBytes("q");
511     Put put = new Put(key);
512     put.setDurability(Durability.SKIP_WAL);
513     for (byte[] family: families) {
514       put.add(family, q, value);
515     }
516     table.put(put);
517   }
518 
519   public static void deleteAllSnapshots(final HBaseAdmin admin)
520       throws IOException {
521     // Delete all the snapshots
522     for (SnapshotDescription snapshot: admin.listSnapshots()) {
523       admin.deleteSnapshot(snapshot.getName());
524     }
525     SnapshotTestingUtils.assertNoSnapshots(admin);
526   }
527 
528   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
529       throws IOException {
530     // Ensure the archiver to be empty
531     MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
532     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
533     mfs.getFileSystem().delete(archiveDir, true);
534   }
535 
536   public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
537       long expectedRows) throws IOException {
538     HTable table = new HTable(util.getConfiguration(), tableName);
539     try {
540       assertEquals(expectedRows, util.countRows(table));
541     } finally {
542       table.close();
543     }
544   }
545 }