View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.util.HashSet;
23  import java.util.TreeMap;
24  import java.util.LinkedList;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.Set;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  
32  import org.apache.hadoop.classification.InterfaceAudience;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.FileStatus;
36  
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.io.Reference;
39  import org.apache.hadoop.hbase.io.HFileLink;
40  import org.apache.hadoop.hbase.regionserver.wal.HLog;
41  import org.apache.hadoop.hbase.util.FSUtils;
42  import org.apache.hadoop.hbase.util.FSVisitor;
43  
44  /**
45   * Utility methods for interacting with the snapshot referenced files.
46   */
47  @InterfaceAudience.Private
48  public final class SnapshotReferenceUtil {
49    public interface FileVisitor extends FSVisitor.StoreFileVisitor,
50      FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
51    }
52  
53    private SnapshotReferenceUtil() {
54      // private constructor for utility class
55    }
56  
57    /**
58     * Get log directory for a server in a snapshot.
59     *
60     * @param snapshotDir directory where the specific snapshot is stored
61     * @param serverName name of the parent regionserver for the log files
62     * @return path to the log home directory for the archive files.
63     */
64    public static Path getLogsDir(Path snapshotDir, String serverName) {
65      return new Path(snapshotDir, HLog.getHLogDirectoryName(serverName));
66    }
67  
68    /**
69     * Get the snapshotted recovered.edits dir for the specified region.
70     *
71     * @param snapshotDir directory where the specific snapshot is stored
72     * @param regionName name of the region
73     * @return path to the recovered.edits directory for the specified region files.
74     */
75    public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
76      return HLog.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
77    }
78  
79    /**
80     * Get the snapshot recovered.edits file
81     *
82     * @param snapshotDir directory where the specific snapshot is stored
83     * @param regionName name of the region
84     * @param logfile name of the edit file
85     * @return full path of the log file for the specified region files.
86     */
87    public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
88      return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
89    }
90  
91    /**
92     * Iterate over the snapshot store files, restored.edits and logs
93     *
94     * @param fs {@link FileSystem}
95     * @param snapshotDir {@link Path} to the Snapshot directory
96     * @param visitor callback object to get the referenced files
97     * @throws IOException if an error occurred while scanning the directory
98     */
99    public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
100       final FileVisitor visitor) throws IOException {
101     visitTableStoreFiles(fs, snapshotDir, visitor);
102     visitRecoveredEdits(fs, snapshotDir, visitor);
103     visitLogFiles(fs, snapshotDir, visitor);
104   }
105 
106   /**
107    * Iterate over the snapshot store files
108    *
109    * @param fs {@link FileSystem}
110    * @param snapshotDir {@link Path} to the Snapshot directory
111    * @param visitor callback object to get the store files
112    * @throws IOException if an error occurred while scanning the directory
113    */
114   public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
115       final FSVisitor.StoreFileVisitor visitor) throws IOException {
116     FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
117   }
118 
119   /**
120    * Iterate over the snapshot store files in the specified region
121    *
122    * @param fs {@link FileSystem}
123    * @param regionDir {@link Path} to the Snapshot region directory
124    * @param visitor callback object to get the store files
125    * @throws IOException if an error occurred while scanning the directory
126    */
127   public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
128       final FSVisitor.StoreFileVisitor visitor) throws IOException {
129     FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
130   }
131 
132   /**
133    * Iterate over the snapshot recovered.edits
134    *
135    * @param fs {@link FileSystem}
136    * @param snapshotDir {@link Path} to the Snapshot directory
137    * @param visitor callback object to get the recovered.edits files
138    * @throws IOException if an error occurred while scanning the directory
139    */
140   public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
141       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
142     FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
143   }
144 
145   /**
146    * Iterate over the snapshot log files
147    *
148    * @param fs {@link FileSystem}
149    * @param snapshotDir {@link Path} to the Snapshot directory
150    * @param visitor callback object to get the log files
151    * @throws IOException if an error occurred while scanning the directory
152    */
153   public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
154       final FSVisitor.LogFileVisitor visitor) throws IOException {
155     FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
156   }
157 
158   /**
159    * Returns the set of region names available in the snapshot.
160    *
161    * @param fs {@link FileSystem}
162    * @param snapshotDir {@link Path} to the Snapshot directory
163    * @throws IOException if an error occurred while scanning the directory
164    * @return the set of the regions contained in the snapshot
165    */
166   public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
167       throws IOException {
168     FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
169     if (regionDirs == null) return null;
170 
171     Set<String> regions = new HashSet<String>();
172     for (FileStatus regionDir: regionDirs) {
173       regions.add(regionDir.getPath().getName());
174     }
175     return regions;
176   }
177 
178   /**
179    * Get the list of hfiles for the specified snapshot region.
180    * NOTE: The current implementation keeps one empty file per HFile in the region.
181    * The file name matches the one in the original table, and by reconstructing
182    * the path you can quickly jump to the referenced file.
183    *
184    * @param fs {@link FileSystem}
185    * @param snapshotRegionDir {@link Path} to the Snapshot region directory
186    * @return Map of hfiles per family, the key is the family name and values are hfile names
187    * @throws IOException if an error occurred while scanning the directory
188    */
189   public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
190       final Path snapshotRegionDir) throws IOException {
191     final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
192 
193     visitRegionStoreFiles(fs, snapshotRegionDir,
194       new FSVisitor.StoreFileVisitor() {
195         public void storeFile (final String region, final String family, final String hfile)
196             throws IOException {
197           List<String> hfiles = familyFiles.get(family);
198           if (hfiles == null) {
199             hfiles = new LinkedList<String>();
200             familyFiles.put(family, hfiles);
201           }
202           hfiles.add(hfile);
203         }
204     });
205 
206     return familyFiles;
207   }
208 
209   /**
210    * Returns the store file names in the snapshot.
211    *
212    * @param fs {@link FileSystem}
213    * @param snapshotDir {@link Path} to the Snapshot directory
214    * @throws IOException if an error occurred while scanning the directory
215    * @return the names of hfiles in the specified snaphot
216    */
217   public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
218       throws IOException {
219     final Set<String> names = new HashSet<String>();
220     visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
221       public void storeFile (final String region, final String family, final String hfile)
222           throws IOException {
223         if (HFileLink.isHFileLink(hfile)) {
224           names.add(HFileLink.getReferencedHFileName(hfile));
225         } else {
226           names.add(hfile);
227         }
228       }
229     });
230     return names;
231   }
232 
233   /**
234    * Returns the log file names available in the snapshot.
235    *
236    * @param fs {@link FileSystem}
237    * @param snapshotDir {@link Path} to the Snapshot directory
238    * @throws IOException if an error occurred while scanning the directory
239    * @return the names of hlogs in the specified snaphot
240    */
241   public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir)
242       throws IOException {
243     final Set<String> names = new HashSet<String>();
244     visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
245       public void logFile (final String server, final String logfile) throws IOException {
246         names.add(logfile);
247       }
248     });
249     return names;
250   }
251 }