View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.Collection;
28  import java.util.List;
29  import java.util.Set;
30  import java.util.HashSet;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.fs.PathFilter;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.TableName;
45  import org.apache.hadoop.hbase.TableNotEnabledException;
46  import org.apache.hadoop.hbase.client.Durability;
47  import org.apache.hadoop.hbase.client.HBaseAdmin;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.Put;
50  import org.apache.hadoop.hbase.io.HFileLink;
51  import org.apache.hadoop.hbase.master.HMaster;
52  import org.apache.hadoop.hbase.master.MasterFileSystem;
53  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
54  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
55  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
56  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
57  import org.apache.hadoop.hbase.regionserver.HRegion;
58  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
59  import org.apache.hadoop.hbase.regionserver.HRegionServer;
60  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.FSTableDescriptors;
63  import org.apache.hadoop.hbase.util.FSUtils;
64  import org.apache.hadoop.hbase.util.FSVisitor;
65  import org.apache.hadoop.hbase.util.MD5Hash;
66  import org.junit.Assert;
67  
68  import com.google.protobuf.ServiceException;
69  
70  /**
71   * Utilities class for snapshots
72   */
73  public class SnapshotTestingUtils {
74  
75    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
76    private static byte[] KEYS = Bytes.toBytes("0123456789");
77  
78    /**
79     * Assert that we don't have any snapshots lists
80     *
81     * @throws IOException
82     *           if the admin operation fails
83     */
84    public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
85      assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
86          .size());
87    }
88  
89    /**
90     * Make sure that there is only one snapshot returned from the master and its
91     * name and table match the passed in parameters.
92     */
93    public static List<SnapshotDescription> assertExistsMatchingSnapshot(
94        HBaseAdmin admin, String snapshotName, TableName tableName)
95        throws IOException {
96      // list the snapshot
97      List<SnapshotDescription> snapshots = admin.listSnapshots();
98  
99      List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
100     for (SnapshotDescription sd : snapshots) {
101       if (snapshotName.equals(sd.getName()) &&
102           tableName.equals(TableName.valueOf(sd.getTable()))) {
103         returnedSnapshots.add(sd);
104       }
105     }
106 
107     Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
108     return returnedSnapshots;
109   }
110 
111   /**
112    * Make sure that there is only one snapshot returned from the master
113    */
114   public static void assertOneSnapshotThatMatches(HBaseAdmin admin,
115       SnapshotDescription snapshot) throws IOException {
116     assertOneSnapshotThatMatches(admin, snapshot.getName(),
117         TableName.valueOf(snapshot.getTable()));
118   }
119 
120   /**
121    * Make sure that there is only one snapshot returned from the master and its
122    * name and table match the passed in parameters.
123    */
124   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
125       HBaseAdmin admin, String snapshotName, TableName tableName)
126       throws IOException {
127     // list the snapshot
128     List<SnapshotDescription> snapshots = admin.listSnapshots();
129 
130     assertEquals("Should only have 1 snapshot", 1, snapshots.size());
131     assertEquals(snapshotName, snapshots.get(0).getName());
132     assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
133 
134     return snapshots;
135   }
136 
137   /**
138    * Make sure that there is only one snapshot returned from the master and its
139    * name and table match the passed in parameters.
140    */
141   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
142       HBaseAdmin admin, byte[] snapshot, TableName tableName) throws IOException {
143     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
144         tableName);
145   }
146 
147   /**
148    * Confirm that the snapshot contains references to all the files that should
149    * be in the snapshot.
150    */
151   public static void confirmSnapshotValid(
152       SnapshotDescription snapshotDescriptor, TableName tableName,
153       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
154       boolean requireLogs, Path logsDir, Set<String> snapshotServers)
155       throws IOException {
156     ArrayList nonEmptyTestFamilies = new ArrayList(1);
157     nonEmptyTestFamilies.add(testFamily);
158     confirmSnapshotValid(snapshotDescriptor, tableName,
159       nonEmptyTestFamilies, null, rootDir, admin, fs, requireLogs,
160       logsDir, snapshotServers);
161   }
162 
163   /**
164    * Confirm that the snapshot has no references files but only metadata.
165    */
166   public static void confirmEmptySnapshotValid(
167       SnapshotDescription snapshotDescriptor, TableName tableName,
168       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
169       boolean requireLogs, Path logsDir, Set<String> snapshotServers)
170       throws IOException {
171     ArrayList emptyTestFamilies = new ArrayList(1);
172     emptyTestFamilies.add(testFamily);
173     confirmSnapshotValid(snapshotDescriptor, tableName,
174       null, emptyTestFamilies, rootDir, admin, fs, requireLogs,
175       logsDir, snapshotServers);
176   }
177 
178   /**
179    * Confirm that the snapshot contains references to all the files that should
180    * be in the snapshot. This method also perform some redundant check like
181    * the existence of the snapshotinfo or the regioninfo which are done always
182    * by the MasterSnapshotVerifier, at the end of the snapshot operation.
183    */
184   public static void confirmSnapshotValid(
185       SnapshotDescription snapshotDescriptor, TableName tableName,
186       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
187       Path rootDir, HBaseAdmin admin, FileSystem fs, boolean requireLogs,
188       Path logsDir, Set<String> snapshotServers) throws IOException {
189     // check snapshot dir
190     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
191         snapshotDescriptor, rootDir);
192     assertTrue(fs.exists(snapshotDir));
193 
194     // check snapshot info
195     Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
196     assertTrue(fs.exists(snapshotinfo));
197 
198     // check the logs dir
199     if (requireLogs) {
200       TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir,
201           snapshotServers, snapshotDescriptor, new Path(snapshotDir,
202               HConstants.HREGION_LOGDIR_NAME));
203     }
204 
205     // check the table info
206     HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, tableName);
207     HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
208     assertEquals(desc, snapshotDesc);
209 
210     // Extract regions and families with store files
211     final Set<String> snapshotRegions = new HashSet<String>();
212     final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
213     FSVisitor.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
214       public void storeFile(final String region, final String family, final String hfileName)
215           throws IOException {
216         snapshotRegions.add(region);
217         snapshotFamilies.add(Bytes.toBytes(family));
218       }
219     });
220 
221     // Verify that there are store files in the specified families
222     if (nonEmptyTestFamilies != null) {
223       for (final byte[] familyName: nonEmptyTestFamilies) {
224         assertTrue(snapshotFamilies.contains(familyName));
225       }
226     }
227 
228     // Verify that there are no store files in the specified families
229     if (emptyTestFamilies != null) {
230       for (final byte[] familyName: emptyTestFamilies) {
231         assertFalse(snapshotFamilies.contains(familyName));
232       }
233     }
234 
235     // Avoid checking regions if the request is for an empty snapshot
236     if ((nonEmptyTestFamilies == null || nonEmptyTestFamilies.size() == 0) &&
237         (emptyTestFamilies != null && emptyTestFamilies.size() > 0)) {
238       assertEquals(0, snapshotRegions.size());
239       return;
240     }
241 
242     // check the region snapshot for all the regions
243     List<HRegionInfo> regions = admin.getTableRegions(tableName);
244     assertEquals(regions.size(), snapshotRegions.size());
245 
246     // Verify Regions
247     for (HRegionInfo info : regions) {
248       String regionName = info.getEncodedName();
249       assertTrue(snapshotRegions.contains(regionName));
250 
251       Path regionDir = new Path(snapshotDir, regionName);
252       HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
253       assertEquals(info, snapshotRegionInfo);
254     }
255   }
256 
257   /**
258    * Helper method for testing async snapshot operations. Just waits for the
259    * given snapshot to complete on the server by repeatedly checking the master.
260    *
261    * @param master: the master running the snapshot
262    * @param snapshot: the snapshot to check
263    * @param sleep: amount to sleep between checks to see if the snapshot is done
264    * @throws ServiceException if the snapshot fails
265    */
266   public static void waitForSnapshotToComplete(HMaster master,
267       SnapshotDescription snapshot, long sleep) throws ServiceException {
268     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
269         .setSnapshot(snapshot).build();
270     IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
271         .buildPartial();
272     while (!done.getDone()) {
273       done = master.isSnapshotDone(null, request);
274       try {
275         Thread.sleep(sleep);
276       } catch (InterruptedException e) {
277         throw new ServiceException(e);
278       }
279     }
280   }
281 
282   public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName)
283       throws IOException {
284     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
285   }
286 
287   public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName)
288       throws IOException {
289     // delete the taken snapshot
290     admin.deleteSnapshot(snapshotName);
291     assertNoSnapshots(admin);
292   }
293 
294   /**
295    * Expect the snapshot to throw an error when checking if the snapshot is
296    * complete
297    *
298    * @param master master to check
299    * @param snapshot the {@link SnapshotDescription} request to pass to the master
300    * @param clazz expected exception from the master
301    */
302   public static void expectSnapshotDoneException(HMaster master,
303       IsSnapshotDoneRequest snapshot,
304       Class<? extends HBaseSnapshotException> clazz) {
305     try {
306       master.isSnapshotDone(null, snapshot);
307       Assert.fail("didn't fail to lookup a snapshot");
308     } catch (ServiceException se) {
309       try {
310         throw ProtobufUtil.getRemoteException(se);
311       } catch (HBaseSnapshotException e) {
312         assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
313       } catch (Throwable t) {
314         Assert.fail("Threw an unexpected exception:" + t);
315       }
316     }
317   }
318 
319   /**
320    * List all the HFiles in the given table
321    *
322    * @param fs: FileSystem where the table lives
323    * @param tableDir directory of the table
324    * @return array of the current HFiles in the table (could be a zero-length array)
325    * @throws IOException on unexecpted error reading the FS
326    */
327   public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
328       throws IOException {
329     final ArrayList<Path> hfiles = new ArrayList<Path>();
330     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
331       public void storeFile(final String region, final String family, final String hfileName)
332           throws IOException {
333         hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
334       }
335     });
336     return hfiles.toArray(new Path[hfiles.size()]);
337   }
338 
339   /**
340    * Take a snapshot of the specified table and verify that the given family is
341    * not empty. Note that this will leave the table disabled
342    * in the case of an offline snapshot.
343    */
344   public static void createSnapshotAndValidate(HBaseAdmin admin,
345       TableName tableName, String familyName, String snapshotNameString,
346       Path rootDir, FileSystem fs, boolean onlineSnapshot)
347       throws Exception {
348     ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
349     nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
350     createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
351                               snapshotNameString, rootDir, fs, onlineSnapshot);
352   }
353 
354   /**
355    * Take a snapshot of the specified table and verify the given families.
356    * Note that this will leave the table disabled in the case of an offline snapshot.
357    */
358   public static void createSnapshotAndValidate(HBaseAdmin admin,
359       TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
360       String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
361         throws Exception {
362     if (!onlineSnapshot) {
363       try {
364         admin.disableTable(tableName);
365       } catch (TableNotEnabledException tne) {
366         LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
367             "already disabled.");
368       }
369     }
370     admin.snapshot(snapshotNameString, tableName);
371 
372     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
373       snapshotNameString, tableName);
374     if (snapshots == null || snapshots.size() != 1) {
375       Assert.fail("Incorrect number of snapshots for table " + tableName);
376     }
377 
378     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
379       emptyFamilyNames, rootDir, admin, fs, false,
380       new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), null);
381   }
382 
383   /**
384    * Corrupt the specified snapshot by deleting some files.
385    *
386    * @param util {@link HBaseTestingUtility}
387    * @param snapshotName name of the snapshot to corrupt
388    * @return array of the corrupted HFiles
389    * @throws IOException on unexecpted error reading the FS
390    */
391   public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
392       throws IOException {
393     final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
394     final FileSystem fs = mfs.getFileSystem();
395 
396     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
397                                                                         mfs.getRootDir());
398     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
399     final TableName table = TableName.valueOf(snapshotDesc.getTable());
400 
401     final ArrayList corruptedFiles = new ArrayList();
402     SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
403       public void storeFile (final String region, final String family, final String hfile)
404           throws IOException {
405         HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile);
406         if (corruptedFiles.size() % 2 == 0) {
407           fs.delete(link.getAvailablePath(fs));
408           corruptedFiles.add(hfile);
409         }
410       }
411     });
412 
413     assertTrue(corruptedFiles.size() > 0);
414     return corruptedFiles;
415   }
416 
417   // ==========================================================================
418   //  Table Helpers
419   // ==========================================================================
420   public static void waitForTableToBeOnline(final HBaseTestingUtility util,
421                                             final TableName tableName)
422       throws IOException, InterruptedException {
423     HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
424     List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
425     for (HRegion region : onlineRegions) {
426       region.waitForFlushesAndCompactions();
427     }
428     util.getHBaseAdmin().isTableAvailable(tableName);
429   }
430 
431   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
432       final byte[]... families) throws IOException, InterruptedException {
433     HTableDescriptor htd = new HTableDescriptor(tableName);
434     for (byte[] family: families) {
435       HColumnDescriptor hcd = new HColumnDescriptor(family);
436       htd.addFamily(hcd);
437     }
438     byte[][] splitKeys = new byte[KEYS.length-2][];
439     for (int i = 0; i < splitKeys.length; ++i) {
440       splitKeys[i] = new byte[] { KEYS[i+1] };
441     }
442     util.getHBaseAdmin().createTable(htd, splitKeys);
443     waitForTableToBeOnline(util, tableName);
444     assertEquals(KEYS.length-1, util.getHBaseAdmin().getTableRegions(tableName).size());
445   }
446 
447   public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
448       byte[]... families) throws IOException, InterruptedException {
449     loadData(util, new HTable(util.getConfiguration(), tableName), rows, families);
450   }
451 
452   public static void loadData(final HBaseTestingUtility util, final HTable table, int rows,
453       byte[]... families) throws IOException, InterruptedException {
454     table.setAutoFlush(false, true);
455 
456     // Ensure one row per region
457     assertTrue(rows >= KEYS.length);
458     for (byte k0: KEYS) {
459       byte[] k = new byte[] { k0 };
460       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
461       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
462       putData(table, families, key, value);
463       rows--;
464     }
465 
466     // Add other extra rows. more rows, more files
467     while (rows-- > 0) {
468       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
469       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
470       putData(table, families, key, value);
471     }
472     table.flushCommits();
473 
474     waitForTableToBeOnline(util, table.getName());
475   }
476 
477   private static void putData(final HTable table, final byte[][] families,
478       final byte[] key, final byte[] value) throws IOException {
479     byte[] q = Bytes.toBytes("q");
480     Put put = new Put(key);
481     put.setDurability(Durability.SKIP_WAL);
482     for (byte[] family: families) {
483       put.add(family, q, value);
484     }
485     table.put(put);
486   }
487 
488   public static void deleteAllSnapshots(final HBaseAdmin admin)
489       throws IOException {
490     // Delete all the snapshots
491     for (SnapshotDescription snapshot: admin.listSnapshots()) {
492       admin.deleteSnapshot(snapshot.getName());
493     }
494     SnapshotTestingUtils.assertNoSnapshots(admin);
495   }
496 
497   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
498       throws IOException {
499     // Ensure the archiver to be empty
500     MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
501     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
502     mfs.getFileSystem().delete(archiveDir, true);
503   }
504 
505   public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
506       long expectedRows) throws IOException {
507     HTable table = new HTable(util.getConfiguration(), tableName);
508     try {
509       assertEquals(expectedRows, util.countRows(table));
510     } finally {
511       table.close();
512     }
513   }
514 }