1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.util.HashSet;
24 import java.util.TreeMap;
25 import java.util.LinkedList;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.Set;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.classification.InterfaceAudience;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.FileStatus;
38
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.io.Reference;
41 import org.apache.hadoop.hbase.io.HFileLink;
42 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
43 import org.apache.hadoop.hbase.regionserver.wal.HLog;
44 import org.apache.hadoop.hbase.util.FSUtils;
45 import org.apache.hadoop.hbase.util.FSVisitor;
46
47
48
49
50 @InterfaceAudience.Private
51 public final class SnapshotReferenceUtil {
52 public interface FileVisitor extends FSVisitor.StoreFileVisitor,
53 FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
54 }
55
56 private SnapshotReferenceUtil() {
57
58 }
59
60
61
62
63
64
65
66
67 public static Path getLogsDir(Path snapshotDir, String serverName) {
68 return new Path(snapshotDir, HLog.getHLogDirectoryName(serverName));
69 }
70
71
72
73
74
75
76
77
78 public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
79 return HLog.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
80 }
81
82
83
84
85
86
87
88
89
90 public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
91 return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
92 }
93
94
95
96
97
98
99
100
101
102 public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
103 final FileVisitor visitor) throws IOException {
104 visitTableStoreFiles(fs, snapshotDir, visitor);
105 visitRecoveredEdits(fs, snapshotDir, visitor);
106 visitLogFiles(fs, snapshotDir, visitor);
107 }
108
109
110
111
112
113
114
115
116
117 public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
118 final FSVisitor.StoreFileVisitor visitor) throws IOException {
119 FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
120 }
121
122
123
124
125
126
127
128
129
130 public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
131 final FSVisitor.StoreFileVisitor visitor) throws IOException {
132 FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
133 }
134
135
136
137
138
139
140
141
142
143 public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
144 final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
145 FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
146 }
147
148
149
150
151
152
153
154
155
156 public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
157 final FSVisitor.LogFileVisitor visitor) throws IOException {
158 FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
159 }
160
161
162
163
164
165
166
167
168
169
170
171 public static void verifySnapshot(final Configuration conf, final FileSystem fs,
172 final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException {
173 final String table = snapshotDesc.getTable();
174 visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
175 public void storeFile (final String region, final String family, final String hfile)
176 throws IOException {
177 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
178 try {
179 link.getFileStatus(fs);
180 } catch (FileNotFoundException e) {
181 throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e);
182 }
183 }
184 });
185 }
186
187
188
189
190
191
192
193
194
195 public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
196 throws IOException {
197 FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
198 if (regionDirs == null) return null;
199
200 Set<String> regions = new HashSet<String>();
201 for (FileStatus regionDir: regionDirs) {
202 regions.add(regionDir.getPath().getName());
203 }
204 return regions;
205 }
206
207
208
209
210
211
212
213
214
215
216
217
218 public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
219 final Path snapshotRegionDir) throws IOException {
220 final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
221
222 visitRegionStoreFiles(fs, snapshotRegionDir,
223 new FSVisitor.StoreFileVisitor() {
224 public void storeFile (final String region, final String family, final String hfile)
225 throws IOException {
226 List<String> hfiles = familyFiles.get(family);
227 if (hfiles == null) {
228 hfiles = new LinkedList<String>();
229 familyFiles.put(family, hfiles);
230 }
231 hfiles.add(hfile);
232 }
233 });
234
235 return familyFiles;
236 }
237
238
239
240
241
242
243
244
245
246 public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
247 throws IOException {
248 final Set<String> names = new HashSet<String>();
249 visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
250 public void storeFile (final String region, final String family, final String hfile)
251 throws IOException {
252 if (HFileLink.isHFileLink(hfile)) {
253 names.add(HFileLink.getReferencedHFileName(hfile));
254 } else {
255 names.add(hfile);
256 }
257 }
258 });
259 return names;
260 }
261
262
263
264
265
266
267
268
269
270 public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir)
271 throws IOException {
272 final Set<String> names = new HashSet<String>();
273 visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
274 public void logFile (final String server, final String logfile) throws IOException {
275 names.add(logfile);
276 }
277 });
278 return names;
279 }
280 }