View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.Collection;
28  import java.util.List;
29  import java.util.Set;
30  import java.util.HashSet;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.fs.PathFilter;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.TableName;
45  import org.apache.hadoop.hbase.TableNotEnabledException;
46  import org.apache.hadoop.hbase.client.Durability;
47  import org.apache.hadoop.hbase.client.HBaseAdmin;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.Put;
50  import org.apache.hadoop.hbase.io.HFileLink;
51  import org.apache.hadoop.hbase.master.HMaster;
52  import org.apache.hadoop.hbase.master.MasterFileSystem;
53  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
54  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
55  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
56  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
57  import org.apache.hadoop.hbase.regionserver.HRegion;
58  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
59  import org.apache.hadoop.hbase.regionserver.HRegionServer;
60  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.FSTableDescriptors;
63  import org.apache.hadoop.hbase.util.FSUtils;
64  import org.apache.hadoop.hbase.util.FSVisitor;
65  import org.apache.hadoop.hbase.util.MD5Hash;
66  import org.junit.Assert;
67  
68  import com.google.protobuf.ServiceException;
69  
70  /**
71   * Utilities class for snapshots
72   */
73  public class SnapshotTestingUtils {
74  
75    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
76  
77    /**
78     * Assert that we don't have any snapshots lists
79     *
80     * @throws IOException
81     *           if the admin operation fails
82     */
83    public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
84      assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
85          .size());
86    }
87  
88    /**
89     * Make sure that there is only one snapshot returned from the master and its
90     * name and table match the passed in parameters.
91     */
92    public static List<SnapshotDescription> assertExistsMatchingSnapshot(
93        HBaseAdmin admin, String snapshotName, TableName tableName)
94        throws IOException {
95      // list the snapshot
96      List<SnapshotDescription> snapshots = admin.listSnapshots();
97  
98      List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
99      for (SnapshotDescription sd : snapshots) {
100       if (snapshotName.equals(sd.getName()) &&
101           tableName.equals(TableName.valueOf(sd.getTable()))) {
102         returnedSnapshots.add(sd);
103       }
104     }
105 
106     Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
107     return returnedSnapshots;
108   }
109 
110   /**
111    * Make sure that there is only one snapshot returned from the master
112    */
113   public static void assertOneSnapshotThatMatches(HBaseAdmin admin,
114       SnapshotDescription snapshot) throws IOException {
115     assertOneSnapshotThatMatches(admin, snapshot.getName(),
116         TableName.valueOf(snapshot.getTable()));
117   }
118 
119   /**
120    * Make sure that there is only one snapshot returned from the master and its
121    * name and table match the passed in parameters.
122    */
123   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
124       HBaseAdmin admin, String snapshotName, TableName tableName)
125       throws IOException {
126     // list the snapshot
127     List<SnapshotDescription> snapshots = admin.listSnapshots();
128 
129     assertEquals("Should only have 1 snapshot", 1, snapshots.size());
130     assertEquals(snapshotName, snapshots.get(0).getName());
131     assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
132 
133     return snapshots;
134   }
135 
136   /**
137    * Make sure that there is only one snapshot returned from the master and its
138    * name and table match the passed in parameters.
139    */
140   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
141       HBaseAdmin admin, byte[] snapshot, TableName tableName) throws IOException {
142     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
143         tableName);
144   }
145 
146   /**
147    * Confirm that the snapshot contains references to all the files that should
148    * be in the snapshot.
149    */
150   public static void confirmSnapshotValid(
151       SnapshotDescription snapshotDescriptor, TableName tableName,
152       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
153       boolean requireLogs, Path logsDir, Set<String> snapshotServers)
154       throws IOException {
155     ArrayList nonEmptyTestFamilies = new ArrayList(1);
156     nonEmptyTestFamilies.add(testFamily);
157     confirmSnapshotValid(snapshotDescriptor, tableName,
158       nonEmptyTestFamilies, null, rootDir, admin, fs, requireLogs,
159       logsDir, snapshotServers);
160   }
161 
162   /**
163    * Confirm that the snapshot has no references files but only metadata.
164    */
165   public static void confirmEmptySnapshotValid(
166       SnapshotDescription snapshotDescriptor, TableName tableName,
167       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
168       boolean requireLogs, Path logsDir, Set<String> snapshotServers)
169       throws IOException {
170     ArrayList emptyTestFamilies = new ArrayList(1);
171     emptyTestFamilies.add(testFamily);
172     confirmSnapshotValid(snapshotDescriptor, tableName,
173       null, emptyTestFamilies, rootDir, admin, fs, requireLogs,
174       logsDir, snapshotServers);
175   }
176 
177   /**
178    * Confirm that the snapshot contains references to all the files that should
179    * be in the snapshot. This method also perform some redundant check like
180    * the existence of the snapshotinfo or the regioninfo which are done always
181    * by the MasterSnapshotVerifier, at the end of the snapshot operation.
182    */
183   public static void confirmSnapshotValid(
184       SnapshotDescription snapshotDescriptor, TableName tableName,
185       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
186       Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs,
187       Path logsDir, Set<String> snapshotServers) throws IOException {
188     // check snapshot dir
189     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
190         snapshotDescriptor, rootDir);
191     assertTrue(fs.exists(snapshotDir));
192 
193     // check snapshot info
194     Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
195     assertTrue(fs.exists(snapshotinfo));
196 
197     // check the logs dir
198     if (requireLogs) {
199       TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir,
200           snapshotServers, snapshotDescriptor, new Path(snapshotDir,
201               HConstants.HREGION_LOGDIR_NAME));
202     }
203 
204     // check the table info
205     HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, tableName);
206     HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
207     assertEquals(desc, snapshotDesc);
208 
209     // Extract regions and families with store files
210     final Set<String> snapshotRegions = new HashSet<String>();
211     final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
212     FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
213       public void storeFile(final String region, final String family, final String hfileName)
214           throws IOException {
215         snapshotRegions.add(region);
216         snapshotFamilies.add(Bytes.toBytes(family));
217       }
218     });
219 
220     // Verify that there are store files in the specified families
221     if (nonEmptyTestFamilies != null) {
222       for (final byte[] familyName: nonEmptyTestFamilies) {
223         assertTrue(snapshotFamilies.contains(familyName));
224       }
225     }
226 
227     // Verify that there are no store files in the specified families
228     if (emptyTestFamilies != null) {
229       for (final byte[] familyName: emptyTestFamilies) {
230         assertFalse(snapshotFamilies.contains(familyName));
231       }
232     }
233 
234     // Avoid checking regions if the request is for an empty snapshot
235     if ((nonEmptyTestFamilies == null || nonEmptyTestFamilies.size() == 0) &&
236         (emptyTestFamilies != null && emptyTestFamilies.size() > 0)) {
237       assertEquals(0, snapshotRegions.size());
238       return;
239     }
240 
241     // check the region snapshot for all the regions
242     List<HRegionInfo> regions = admin.getTableRegions(tableName);
243     assertEquals(regions.size(), snapshotRegions.size());
244 
245     // Verify Regions
246     for (HRegionInfo info : regions) {
247       String regionName = info.getEncodedName();
248       assertTrue(snapshotRegions.contains(regionName));
249 
250       Path regionDir = new Path(snapshotDir, regionName);
251       HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
252       assertEquals(info, snapshotRegionInfo);
253     }
254   }
255 
256   /**
257    * Helper method for testing async snapshot operations. Just waits for the
258    * given snapshot to complete on the server by repeatedly checking the master.
259    *
260    * @param master: the master running the snapshot
261    * @param snapshot: the snapshot to check
262    * @param sleep: amount to sleep between checks to see if the snapshot is done
263    * @throws ServiceException if the snapshot fails
264    */
265   public static void waitForSnapshotToComplete(HMaster master,
266       SnapshotDescription snapshot, long sleep) throws ServiceException {
267     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
268         .setSnapshot(snapshot).build();
269     IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
270         .buildPartial();
271     while (!done.getDone()) {
272       done = master.isSnapshotDone(null, request);
273       try {
274         Thread.sleep(sleep);
275       } catch (InterruptedException e) {
276         throw new ServiceException(e);
277       }
278     }
279   }
280 
281   /*
282    * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
283    * except for the last CorruptedSnapshotException
284    */
285   public static void snapshot(HBaseAdmin admin,
286       final String snapshotName, final String tableName,
287       SnapshotDescription.Type type, int numTries) throws IOException {
288     int tries = 0;
289     CorruptedSnapshotException lastEx = null;
290     while (tries++ < numTries) {
291       try {
292         admin.snapshot(snapshotName, tableName, type);
293         return;
294       } catch (CorruptedSnapshotException cse) {
295         LOG.warn("Got CorruptedSnapshotException", cse);
296         lastEx = cse;
297       }
298     }
299     throw lastEx;
300   }
301 
302   public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName)
303       throws IOException {
304     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
305   }
306 
307   public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName)
308       throws IOException {
309     // delete the taken snapshot
310     admin.deleteSnapshot(snapshotName);
311     assertNoSnapshots(admin);
312   }
313 
314   /**
315    * Expect the snapshot to throw an error when checking if the snapshot is
316    * complete
317    *
318    * @param master master to check
319    * @param snapshot the {@link SnapshotDescription} request to pass to the master
320    * @param clazz expected exception from the master
321    */
322   public static void expectSnapshotDoneException(HMaster master,
323       IsSnapshotDoneRequest snapshot,
324       Class<? extends HBaseSnapshotException> clazz) {
325     try {
326       master.isSnapshotDone(null, snapshot);
327       Assert.fail("didn't fail to lookup a snapshot");
328     } catch (ServiceException se) {
329       try {
330         throw ProtobufUtil.getRemoteException(se);
331       } catch (HBaseSnapshotException e) {
332         assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
333       } catch (Throwable t) {
334         Assert.fail("Threw an unexpected exception:" + t);
335       }
336     }
337   }
338 
339   /**
340    * List all the HFiles in the given table
341    *
342    * @param fs: FileSystem where the table lives
343    * @param tableDir directory of the table
344    * @return array of the current HFiles in the table (could be a zero-length array)
345    * @throws IOException on unexecpted error reading the FS
346    */
347   public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
348       throws IOException {
349     final ArrayList<Path> hfiles = new ArrayList<Path>();
350     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
351       public void storeFile(final String region, final String family, final String hfileName)
352           throws IOException {
353         hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
354       }
355     });
356     return hfiles.toArray(new Path[hfiles.size()]);
357   }
358 
359   /**
360    * Take a snapshot of the specified table and verify that the given family is
361    * not empty. Note that this will leave the table disabled
362    * in the case of an offline snapshot.
363    */
364   public static void createSnapshotAndValidate(HBaseAdmin admin,
365       TableName tableName, String familyName, String snapshotNameString,
366       Path rootDir, FileSystem fs, boolean onlineSnapshot)
367       throws Exception {
368     ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
369     nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
370     createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
371                               snapshotNameString, rootDir, fs, onlineSnapshot);
372   }
373 
374   /**
375    * Take a snapshot of the specified table and verify the given families.
376    * Note that this will leave the table disabled in the case of an offline snapshot.
377    */
378   public static void createSnapshotAndValidate(HBaseAdmin admin,
379       TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
380       String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
381         throws Exception {
382     if (!onlineSnapshot) {
383       try {
384         admin.disableTable(tableName);
385       } catch (TableNotEnabledException tne) {
386         LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
387             "already disabled.");
388       }
389     }
390     admin.snapshot(snapshotNameString, tableName);
391 
392     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
393       snapshotNameString, tableName);
394     if (snapshots == null || snapshots.size() != 1) {
395       Assert.fail("Incorrect number of snapshots for table " + tableName);
396     }
397 
398     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
399       emptyFamilyNames, rootDir, admin, fs, false,
400       new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
401   }
402 
403   /**
404    * Corrupt the specified snapshot by deleting some files.
405    *
406    * @param util {@link HBaseTestingUtility}
407    * @param snapshotName name of the snapshot to corrupt
408    * @return array of the corrupted HFiles
409    * @throws IOException on unexecpted error reading the FS
410    */
411   public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
412       throws IOException {
413     final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
414     final FileSystem fs = mfs.getFileSystem();
415 
416     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
417                                                                         mfs.getRootDir());
418     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
419     final TableName table = TableName.valueOf(snapshotDesc.getTable());
420 
421     final ArrayList corruptedFiles = new ArrayList();
422     SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
423       public void storeFile (final String region, final String family, final String hfile)
424           throws IOException {
425         HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);
426         if (corruptedFiles.size() % 2 == 0) {
427           fs.delete(link.getAvailablePath(fs));
428           corruptedFiles.add(hfile);
429         }
430       }
431     });
432 
433     assertTrue(corruptedFiles.size() > 0);
434     return corruptedFiles;
435   }
436 
437   // ==========================================================================
438   //  Table Helpers
439   // ==========================================================================
440   public static void waitForTableToBeOnline(final HBaseTestingUtility util,
441                                             final TableName tableName)
442       throws IOException, InterruptedException {
443     HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
444     List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
445     for (HRegion region : onlineRegions) {
446       region.waitForFlushesAndCompactions();
447     }
448     util.getHBaseAdmin().isTableAvailable(tableName);
449   }
450 
451   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
452       final byte[]... families) throws IOException, InterruptedException {
453     HTableDescriptor htd = new HTableDescriptor(tableName);
454     for (byte[] family: families) {
455       HColumnDescriptor hcd = new HColumnDescriptor(family);
456       htd.addFamily(hcd);
457     }
458     byte[][] splitKeys = new byte[14][];
459     byte[] hex = Bytes.toBytes("123456789abcde");
460     for (int i = 0; i < splitKeys.length; ++i) {
461       splitKeys[i] = new byte[] { hex[i] };
462     }
463     util.getHBaseAdmin().createTable(htd, splitKeys);
464     waitForTableToBeOnline(util, tableName);
465     assertEquals(15, util.getHBaseAdmin().getTableRegions(tableName).size());
466   }
467 
468   public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
469       byte[]... families) throws IOException, InterruptedException {
470     loadData(util, new HTable(util.getConfiguration(), tableName), rows, families);
471   }
472 
473   public static void loadData(final HBaseTestingUtility util, final HTable table, int rows,
474       byte[]... families) throws IOException, InterruptedException {
475     table.setAutoFlush(false, true);
476 
477     // Ensure one row per region
478     assertTrue(rows >= 16);
479     for (byte k0: Bytes.toBytes("0123456789abcdef")) {
480       byte[] k = new byte[] { k0 };
481       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
482       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
483       putData(table, families, key, value);
484       rows--;
485     }
486 
487     // Add other extra rows. more rows, more files
488     while (rows-- > 0) {
489       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
490       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
491       putData(table, families, key, value);
492     }
493     table.flushCommits();
494 
495     waitForTableToBeOnline(util, table.getName());
496   }
497 
498   private static void putData(final HTable table, final byte[][] families,
499       final byte[] key, final byte[] value) throws IOException {
500     byte[] q = Bytes.toBytes("q");
501     Put put = new Put(key);
502     put.setDurability(Durability.SKIP_WAL);
503     for (byte[] family: families) {
504       put.add(family, q, value);
505     }
506     table.put(put);
507   }
508 
509   public static void deleteAllSnapshots(final HBaseAdmin admin)
510       throws IOException {
511     // Delete all the snapshots
512     for (SnapshotDescription snapshot: admin.listSnapshots()) {
513       admin.deleteSnapshot(snapshot.getName());
514     }
515     SnapshotTestingUtils.assertNoSnapshots(admin);
516   }
517 
518   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
519       throws IOException {
520     // Ensure the archiver to be empty
521     MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
522     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
523     mfs.getFileSystem().delete(archiveDir, true);
524   }
525 
526   public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
527       long expectedRows) throws IOException {
528     HTable table = new HTable(util.getConfiguration(), tableName);
529     try {
530       assertEquals(expectedRows, util.countRows(table));
531     } finally {
532       table.close();
533     }
534   }
535 }