View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.util.HashSet;
23  import java.util.LinkedList;
24  import java.util.List;
25  import java.util.Map;
26  import java.util.Set;
27  import java.util.TreeMap;
28  
29  import org.apache.hadoop.classification.InterfaceAudience;
30  import org.apache.hadoop.fs.FileStatus;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.io.HFileLink;
34  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
35  import org.apache.hadoop.hbase.util.FSUtils;
36  import org.apache.hadoop.hbase.util.FSVisitor;
37  
38  /**
39   * Utility methods for interacting with the snapshot referenced files.
40   */
41  @InterfaceAudience.Private
42  public final class SnapshotReferenceUtil {
43    public interface FileVisitor extends FSVisitor.StoreFileVisitor,
44      FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
45    }
46  
47    private SnapshotReferenceUtil() {
48      // private constructor for utility class
49    }
50  
51    /**
52     * Get log directory for a server in a snapshot.
53     *
54     * @param snapshotDir directory where the specific snapshot is stored
55     * @param serverName name of the parent regionserver for the log files
56     * @return path to the log home directory for the archive files.
57     */
58    public static Path getLogsDir(Path snapshotDir, String serverName) {
59      return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
60    }
61  
62    /**
63     * Get the snapshotted recovered.edits dir for the specified region.
64     *
65     * @param snapshotDir directory where the specific snapshot is stored
66     * @param regionName name of the region
67     * @return path to the recovered.edits directory for the specified region files.
68     */
69    public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
70      return HLogUtil.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
71    }
72  
73    /**
74     * Get the snapshot recovered.edits file
75     *
76     * @param snapshotDir directory where the specific snapshot is stored
77     * @param regionName name of the region
78     * @param logfile name of the edit file
79     * @return full path of the log file for the specified region files.
80     */
81    public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
82      return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
83    }
84  
85    /**
86     * Iterate over the snapshot store files, restored.edits and logs
87     *
88     * @param fs {@link FileSystem}
89     * @param snapshotDir {@link Path} to the Snapshot directory
90     * @param visitor callback object to get the referenced files
91     * @throws IOException if an error occurred while scanning the directory
92     */
93    public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
94        final FileVisitor visitor) throws IOException {
95      visitTableStoreFiles(fs, snapshotDir, visitor);
96      visitRecoveredEdits(fs, snapshotDir, visitor);
97      visitLogFiles(fs, snapshotDir, visitor);
98    }
99  
100   /**
101    * Iterate over the snapshot store files
102    *
103    * @param fs {@link FileSystem}
104    * @param snapshotDir {@link Path} to the Snapshot directory
105    * @param visitor callback object to get the store files
106    * @throws IOException if an error occurred while scanning the directory
107    */
108   public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
109       final FSVisitor.StoreFileVisitor visitor) throws IOException {
110     FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
111   }
112 
113   /**
114    * Iterate over the snapshot store files in the specified region
115    *
116    * @param fs {@link FileSystem}
117    * @param regionDir {@link Path} to the Snapshot region directory
118    * @param visitor callback object to get the store files
119    * @throws IOException if an error occurred while scanning the directory
120    */
121   public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
122       final FSVisitor.StoreFileVisitor visitor) throws IOException {
123     FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
124   }
125 
126   /**
127    * Iterate over the snapshot recovered.edits
128    *
129    * @param fs {@link FileSystem}
130    * @param snapshotDir {@link Path} to the Snapshot directory
131    * @param visitor callback object to get the recovered.edits files
132    * @throws IOException if an error occurred while scanning the directory
133    */
134   public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
135       final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
136     FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
137   }
138 
139   /**
140    * Iterate over the snapshot log files
141    *
142    * @param fs {@link FileSystem}
143    * @param snapshotDir {@link Path} to the Snapshot directory
144    * @param visitor callback object to get the log files
145    * @throws IOException if an error occurred while scanning the directory
146    */
147   public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
148       final FSVisitor.LogFileVisitor visitor) throws IOException {
149     FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
150   }
151 
152   /**
153    * Returns the set of region names available in the snapshot.
154    *
155    * @param fs {@link FileSystem}
156    * @param snapshotDir {@link Path} to the Snapshot directory
157    * @throws IOException if an error occurred while scanning the directory
158    * @return the set of the regions contained in the snapshot
159    */
160   public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
161       throws IOException {
162     FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
163     if (regionDirs == null) return null;
164 
165     Set<String> regions = new HashSet<String>();
166     for (FileStatus regionDir: regionDirs) {
167       regions.add(regionDir.getPath().getName());
168     }
169     return regions;
170   }
171 
172   /**
173    * Get the list of hfiles for the specified snapshot region.
174    * NOTE: The current implementation keeps one empty file per HFile in the region.
175    * The file name matches the one in the original table, and by reconstructing
176    * the path you can quickly jump to the referenced file.
177    *
178    * @param fs {@link FileSystem}
179    * @param snapshotRegionDir {@link Path} to the Snapshot region directory
180    * @return Map of hfiles per family, the key is the family name and values are hfile names
181    * @throws IOException if an error occurred while scanning the directory
182    */
183   public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
184       final Path snapshotRegionDir) throws IOException {
185     final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
186 
187     visitRegionStoreFiles(fs, snapshotRegionDir,
188       new FSVisitor.StoreFileVisitor() {
189         public void storeFile (final String region, final String family, final String hfile)
190             throws IOException {
191           List<String> hfiles = familyFiles.get(family);
192           if (hfiles == null) {
193             hfiles = new LinkedList<String>();
194             familyFiles.put(family, hfiles);
195           }
196           hfiles.add(hfile);
197         }
198     });
199 
200     return familyFiles;
201   }
202 
203   /**
204    * Returns the store file names in the snapshot.
205    *
206    * @param fs {@link FileSystem}
207    * @param snapshotDir {@link Path} to the Snapshot directory
208    * @throws IOException if an error occurred while scanning the directory
209    * @return the names of hfiles in the specified snaphot
210    */
211   public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
212       throws IOException {
213     final Set<String> names = new HashSet<String>();
214     visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
215       public void storeFile (final String region, final String family, final String hfile)
216           throws IOException {
217         if (HFileLink.isHFileLink(hfile)) {
218           names.add(HFileLink.getReferencedHFileName(hfile));
219         } else {
220           names.add(hfile);
221         }
222       }
223     });
224     return names;
225   }
226 
227   /**
228    * Returns the log file names available in the snapshot.
229    *
230    * @param fs {@link FileSystem}
231    * @param snapshotDir {@link Path} to the Snapshot directory
232    * @throws IOException if an error occurred while scanning the directory
233    * @return the names of hlogs in the specified snaphot
234    */
235   public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir)
236       throws IOException {
237     final Set<String> names = new HashSet<String>();
238     visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
239       public void logFile (final String server, final String logfile) throws IOException {
240         names.add(logfile);
241       }
242     });
243     return names;
244   }
245 }