1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collection;
27  import java.util.List;
28  import java.util.Set;
29  import java.util.TreeSet;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.fs.PathFilter;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.client.HBaseAdmin;
41  import org.apache.hadoop.hbase.master.HMaster;
42  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
45  import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
46  import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.FSTableDescriptors;
49  import org.apache.hadoop.hbase.util.FSUtils;
50  import org.junit.Assert;
51  
52  /**
53   * Utilities class for snapshots
54   */
55  public class SnapshotTestingUtils {
56  
57    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
58  
59    /**
60     * Assert that we don't have any snapshots lists
61     * @throws IOException if the admin operation fails
62     */
63    public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
64      assertEquals("Have some previous snapshots", 0, admin.listSnapshots().size());
65    }
66  
67    /**
68     * Make sure that there is only one snapshot returned from the master and its name and table match
69     * the passed in parameters.
70     */
71    public static void assertOneSnapshotThatMatches(HBaseAdmin admin, HSnapshotDescription snapshot)
72        throws IOException {
73      assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable());
74    }
75  
76    /**
77     * Make sure that there is only one snapshot returned from the master and its name and table match
78     * the passed in parameters.
79     */
80    public static void assertOneSnapshotThatMatches(HBaseAdmin admin, SnapshotDescription snapshot)
81        throws IOException {
82      assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable());
83    }
84  
85    /**
86     * Make sure that there is only one snapshot returned from the master and its name and table match
87     * the passed in parameters.
88     */
89    public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
90        String snapshotName, String tableName) throws IOException {
91      // list the snapshot
92      List<SnapshotDescription> snapshots = admin.listSnapshots();
93  
94      assertEquals("Should only have 1 snapshot", 1, snapshots.size());
95      assertEquals(snapshotName, snapshots.get(0).getName());
96      assertEquals(tableName, snapshots.get(0).getTable());
97  
98      return snapshots;
99    }
100 
101   /**
102    * Make sure that there is only one snapshot returned from the master and its name and table match
103    * the passed in parameters.
104    */
105   public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
106       byte[] snapshot, byte[] tableName) throws IOException {
107     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), Bytes.toString(tableName));
108   }
109 
110   /**
111    * Confirm that the snapshot contains references to all the files that should be in the snapshot
112    */
113   public static void confirmSnapshotValid(SnapshotDescription snapshotDescriptor,
114       byte[] tableName, byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
115       boolean requireLogs, Path logsDir, Set<String> snapshotServers) throws IOException {
116     Path snapshotDir = SnapshotDescriptionUtils
117         .getCompletedSnapshotDir(snapshotDescriptor, rootDir);
118     assertTrue(fs.exists(snapshotDir));
119     Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
120     assertTrue(fs.exists(snapshotinfo));
121     // check the logs dir
122     if (requireLogs) {
123       TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers,
124         snapshotDescriptor, new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME));
125     }
126     // check the table info
127     HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, tableName);
128     HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
129     assertEquals(desc, snapshotDesc);
130 
131     // check the region snapshot for all the regions
132     List<HRegionInfo> regions = admin.getTableRegions(tableName);
133     for (HRegionInfo info : regions) {
134       String regionName = info.getEncodedName();
135       Path regionDir = new Path(snapshotDir, regionName);
136       HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
137       assertEquals(info, snapshotRegionInfo);
138       // check to make sure we have the family
139       Path familyDir = new Path(regionDir, Bytes.toString(testFamily));
140       assertTrue("Expected to find: " + familyDir + ", but it doesn't exist", fs.exists(familyDir));
141       // make sure we have some files references
142       assertTrue(fs.listStatus(familyDir).length > 0);
143     }
144   }
145 
146   /**
147    * Helper method for testing async snapshot operations. Just waits for the given snapshot to
148    * complete on the server by repeatedly checking the master.
149    * @param master running the snapshot
150    * @param snapshot to check
151    * @param sleep amount to sleep between checks to see if the snapshot is done
152    * @throws IOException if the snapshot fails
153    */
154   public static void waitForSnapshotToComplete(HMaster master, HSnapshotDescription snapshot,
155       long sleep) throws IOException {
156     boolean done = false;
157     while (!done) {
158       done = master.isSnapshotDone(snapshot);
159       try {
160         Thread.sleep(sleep);
161       } catch (InterruptedException e) {
162         throw new IOException(e);
163       }
164     }
165   }
166 
167   public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName) throws IOException {
168     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
169   }
170 
171   public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName) throws IOException {
172     // delete the taken snapshot
173     admin.deleteSnapshot(snapshotName);
174     assertNoSnapshots(admin);
175   }
176 
177   /**
178    * Expect the snapshot to throw an error when checking if the snapshot is complete
179    * @param master master to check
180    * @param snapshot the {@link HSnapshotDescription} request to pass to the master
181    * @param clazz expected exception from the master
182    */
183   public static void expectSnapshotDoneException(HMaster master, HSnapshotDescription snapshot,
184       Class<? extends HBaseSnapshotException> clazz) {
185     try {
186       boolean res = master.isSnapshotDone(snapshot);
187       Assert.fail("didn't fail to lookup a snapshot: res=" + res);
188     } catch (HBaseSnapshotException e) {
189       assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
190     } catch (Throwable t) {
191       Assert.fail("Threw an unexpected exception:" + t);
192     }
193   }
194 
195   /**
196    * List all the HFiles in the given table
197    * @param fs FileSystem where the table lives
198    * @param tableDir directory of the table
199    * @return array of the current HFiles in the table (could be a zero-length array)
200    * @throws IOException on unexecpted error reading the FS
201    */
202   public static FileStatus[] listHFiles(final FileSystem fs, Path tableDir) throws IOException {
203     // setup the filters we will need based on the filesystem
204     PathFilter regionFilter = new FSUtils.RegionDirFilter(fs);
205     PathFilter familyFilter = new FSUtils.FamilyDirFilter(fs);
206     final PathFilter fileFilter = new PathFilter() {
207       @Override
208       public boolean accept(Path file) {
209         try {
210           return fs.isFile(file);
211         } catch (IOException e) {
212           return false;
213         }
214       }
215     };
216 
217     FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, regionFilter);
218     // if no regions, then we are done
219     if (regionDirs == null || regionDirs.length == 0) return new FileStatus[0];
220 
221     // go through each of the regions, and add al the hfiles under each family
222     List<FileStatus> regionFiles = new ArrayList<FileStatus>(regionDirs.length);
223     for (FileStatus regionDir : regionDirs) {
224       FileStatus[] fams = FSUtils.listStatus(fs, regionDir.getPath(), familyFilter);
225       // if no families, then we are done again
226       if (fams == null || fams.length == 0) continue;
227       // add all the hfiles under the family
228       regionFiles.addAll(SnapshotTestingUtils.getHFilesInRegion(fams, fs, fileFilter));
229     }
230     FileStatus[] files = new FileStatus[regionFiles.size()];
231     regionFiles.toArray(files);
232     return files;
233   }
234 
235   /**
236    * Get all the hfiles in the region, under the passed set of families
237    * @param families all the family directories under the region
238    * @param fs filesystem where the families live
239    * @param fileFilter filter to only include files
240    * @return collection of all the hfiles under all the passed in families (non-null)
241    * @throws IOException on unexecpted error reading the FS
242    */
243   public static Collection<FileStatus> getHFilesInRegion(FileStatus[] families, FileSystem fs,
244       PathFilter fileFilter) throws IOException {
245     Set<FileStatus> files = new TreeSet<FileStatus>();
246     for (FileStatus family : families) {
247       // get all the hfiles in the family
248       FileStatus[] hfiles = FSUtils.listStatus(fs, family.getPath(), fileFilter);
249       // if no hfiles, then we are done with this family
250       if (hfiles == null || hfiles.length == 0) continue;
251       files.addAll(Arrays.asList(hfiles));
252     }
253     return files;
254   }
255 }