1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.text.SimpleDateFormat;
24 import java.util.Date;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.classification.InterfaceAudience;
32 import org.apache.hadoop.classification.InterfaceStability;
33 import org.apache.hadoop.conf.Configured;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
36 import org.apache.hadoop.util.StringUtils;
37 import org.apache.hadoop.util.Tool;
38 import org.apache.hadoop.util.ToolRunner;
39
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.io.HFileLink;
44 import org.apache.hadoop.hbase.io.HLogLink;
45 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46 import org.apache.hadoop.hbase.util.FSUtils;
47 import org.apache.hadoop.hbase.util.FSTableDescriptors;
48
49
50
51
52
53
54
55
56
57
58 @InterfaceAudience.Public
59 @InterfaceStability.Evolving
60 public final class SnapshotInfo extends Configured implements Tool {
61 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
62
63
64
65
66
67
68
69
70
71 public static class SnapshotStats {
72
73 static class FileInfo {
74 private final boolean inArchive;
75 private final long size;
76
77 FileInfo(final boolean inArchive, final long size) {
78 this.inArchive = inArchive;
79 this.size = size;
80 }
81
82
83 public boolean inArchive() {
84 return this.inArchive;
85 }
86
87
88 public boolean isMissing() {
89 return this.size < 0;
90 }
91
92
93 public long getSize() {
94 return this.size;
95 }
96 }
97
98 private int hfileArchiveCount = 0;
99 private int hfilesMissing = 0;
100 private int hfilesCount = 0;
101 private int logsMissing = 0;
102 private int logsCount = 0;
103 private long hfileArchiveSize = 0;
104 private long hfileSize = 0;
105 private long logSize = 0;
106
107 private final SnapshotDescription snapshot;
108 private final TableName snapshotTable;
109 private final Configuration conf;
110 private final FileSystem fs;
111
112 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
113 {
114 this.snapshot = snapshot;
115 this.snapshotTable = TableName.valueOf(snapshot.getTable());
116 this.conf = conf;
117 this.fs = fs;
118 }
119
120
121 public SnapshotDescription getSnapshotDescription() {
122 return this.snapshot;
123 }
124
125
126 public boolean isSnapshotCorrupted() {
127 return hfilesMissing > 0 || logsMissing > 0;
128 }
129
130
131 public int getStoreFilesCount() {
132 return hfilesCount + hfileArchiveCount;
133 }
134
135
136 public int getArchivedStoreFilesCount() {
137 return hfileArchiveCount;
138 }
139
140
141 public int getLogsCount() {
142 return logsCount;
143 }
144
145
146 public int getMissingStoreFilesCount() {
147 return hfilesMissing;
148 }
149
150
151 public int getMissingLogsCount() {
152 return logsMissing;
153 }
154
155
156 public long getStoreFilesSize() {
157 return hfileSize + hfileArchiveSize;
158 }
159
160
161 public long getSharedStoreFilesSize() {
162 return hfileSize;
163 }
164
165
166 public long getArchivedStoreFileSize() {
167 return hfileArchiveSize;
168 }
169
170
171 public float getSharedStoreFilePercentage() {
172 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
173 }
174
175
176 public long getLogsSize() {
177 return logSize;
178 }
179
180
181
182
183
184
185
186
187 FileInfo addStoreFile(final String region, final String family, final String hfile)
188 throws IOException {
189 TableName table = snapshotTable;
190 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
191 boolean inArchive = false;
192 long size = -1;
193 try {
194 if ((inArchive = fs.exists(link.getArchivePath()))) {
195 size = fs.getFileStatus(link.getArchivePath()).getLen();
196 hfileArchiveSize += size;
197 hfileArchiveCount++;
198 } else {
199 size = link.getFileStatus(fs).getLen();
200 hfileSize += size;
201 hfilesCount++;
202 }
203 } catch (FileNotFoundException e) {
204 hfilesMissing++;
205 }
206 return new FileInfo(inArchive, size);
207 }
208
209
210
211
212
213
214
215 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
216 Path rootDir = FSUtils.getRootDir(conf);
217 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
218 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
219 long size = fs.getFileStatus(path).getLen();
220 logSize += size;
221 logsCount++;
222 return new FileInfo(true, size);
223 }
224
225
226
227
228
229
230
231 FileInfo addLogFile(final String server, final String logfile) throws IOException {
232 HLogLink logLink = new HLogLink(conf, server, logfile);
233 long size = -1;
234 try {
235 size = logLink.getFileStatus(fs).getLen();
236 logSize += size;
237 logsCount++;
238 } catch (FileNotFoundException e) {
239 logsMissing++;
240 }
241 return new FileInfo(false, size);
242 }
243 }
244
245 private FileSystem fs;
246 private Path rootDir;
247
248 private HTableDescriptor snapshotTableDesc;
249 private SnapshotDescription snapshotDesc;
250 private Path snapshotDir;
251
252 @Override
253 public int run(String[] args) throws IOException, InterruptedException {
254 String snapshotName = null;
255 boolean showSchema = false;
256 boolean showFiles = false;
257 boolean showStats = false;
258
259
260 for (int i = 0; i < args.length; i++) {
261 String cmd = args[i];
262 try {
263 if (cmd.equals("-snapshot")) {
264 snapshotName = args[++i];
265 } else if (cmd.equals("-files")) {
266 showFiles = true;
267 showStats = true;
268 } else if (cmd.equals("-stats")) {
269 showStats = true;
270 } else if (cmd.equals("-schema")) {
271 showSchema = true;
272 } else if (cmd.equals("-h") || cmd.equals("--help")) {
273 printUsageAndExit();
274 } else {
275 System.err.println("UNEXPECTED: " + cmd);
276 printUsageAndExit();
277 }
278 } catch (Exception e) {
279 printUsageAndExit();
280 }
281 }
282
283 if (snapshotName == null) {
284 System.err.println("Missing snapshot name!");
285 printUsageAndExit();
286 return 1;
287 }
288
289 Configuration conf = getConf();
290 fs = FileSystem.get(conf);
291 rootDir = FSUtils.getRootDir(conf);
292
293
294 if (!loadSnapshotInfo(snapshotName)) {
295 System.err.println("Snapshot '" + snapshotName + "' not found!");
296 return 1;
297 }
298
299 printInfo();
300 if (showSchema) printSchema();
301 printFiles(showFiles, showStats);
302
303 return 0;
304 }
305
306
307
308
309
310
311 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
312 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
313 if (!fs.exists(snapshotDir)) {
314 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
315 return false;
316 }
317
318 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
319 snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
320 return true;
321 }
322
323
324
325
326 private void printInfo() {
327 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
328 System.out.println("Snapshot Info");
329 System.out.println("----------------------------------------");
330 System.out.println(" Name: " + snapshotDesc.getName());
331 System.out.println(" Type: " + snapshotDesc.getType());
332 System.out.println(" Table: " + snapshotTableDesc.getTableName().getNameAsString());
333 System.out.println(" Format: " + snapshotDesc.getVersion());
334 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
335 System.out.println();
336 }
337
338
339
340
341 private void printSchema() {
342 System.out.println("Table Descriptor");
343 System.out.println("----------------------------------------");
344 System.out.println(snapshotTableDesc.toString());
345 System.out.println();
346 }
347
348
349
350
351
352 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
353 if (showFiles) {
354 System.out.println("Snapshot Files");
355 System.out.println("----------------------------------------");
356 }
357
358
359 final String table = snapshotTableDesc.getTableName().getNameAsString();
360 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
361 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
362 new SnapshotReferenceUtil.FileVisitor() {
363 public void storeFile (final String region, final String family, final String hfile)
364 throws IOException {
365 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
366
367 if (showFiles) {
368 System.out.printf("%8s %s/%s/%s/%s %s%n",
369 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
370 table, region, family, hfile,
371 (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
372 }
373 }
374
375 public void recoveredEdits (final String region, final String logfile)
376 throws IOException {
377 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
378
379 if (showFiles) {
380 System.out.printf("%8s recovered.edits %s on region %s%n",
381 StringUtils.humanReadableInt(info.getSize()), logfile, region);
382 }
383 }
384
385 public void logFile (final String server, final String logfile)
386 throws IOException {
387 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
388
389 if (showFiles) {
390 System.out.printf("%8s log %s on server %s %s%n",
391 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
392 logfile, server,
393 (info.isMissing() ? "(NOT FOUND)" : ""));
394 }
395 }
396 });
397
398
399 System.out.println();
400 if (stats.isSnapshotCorrupted()) {
401 System.out.println("**************************************************************");
402 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
403 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
404 System.out.println("**************************************************************");
405 }
406
407 if (showStats) {
408 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
409 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
410 StringUtils.humanReadableInt(stats.getStoreFilesSize()),
411 stats.getSharedStoreFilePercentage(),
412 StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
413 );
414 System.out.printf("%d Logs, total size %s%n",
415 stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
416 System.out.println();
417 }
418 }
419
420 private void printUsageAndExit() {
421 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
422 System.err.println(" where [options] are:");
423 System.err.println(" -h|-help Show this help and exit.");
424 System.err.println(" -snapshot NAME Snapshot to examine.");
425 System.err.println(" -files Files and logs list.");
426 System.err.println(" -stats Files and logs stats.");
427 System.err.println(" -schema Describe the snapshotted table.");
428 System.err.println();
429 System.err.println("Examples:");
430 System.err.println(" hbase " + getClass() + " \\");
431 System.err.println(" -snapshot MySnapshot -files");
432 System.exit(1);
433 }
434
435
436
437
438
439
440
441 public static SnapshotStats getSnapshotStats(final Configuration conf,
442 final SnapshotDescription snapshot) throws IOException {
443 Path rootDir = FSUtils.getRootDir(conf);
444 FileSystem fs = FileSystem.get(conf);
445 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
446 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
447 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
448 new SnapshotReferenceUtil.FileVisitor() {
449 public void storeFile (final String region, final String family, final String hfile)
450 throws IOException {
451 stats.addStoreFile(region, family, hfile);
452 }
453
454 public void recoveredEdits (final String region, final String logfile) throws IOException {
455 stats.addRecoveredEdits(region, logfile);
456 }
457
458 public void logFile (final String server, final String logfile) throws IOException {
459 stats.addLogFile(server, logfile);
460 }
461 });
462 return stats;
463 }
464
465
466
467
468
469
470
471
472 static int innerMain(final String [] args) throws Exception {
473 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
474 }
475
476 public static void main(String[] args) throws Exception {
477 System.exit(innerMain(args));
478 }
479 }