1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.net.URI;
24 import java.text.SimpleDateFormat;
25 import java.util.concurrent.atomic.AtomicInteger;
26 import java.util.concurrent.atomic.AtomicLong;
27 import java.util.ArrayList;
28 import java.util.Date;
29 import java.util.List;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.classification.InterfaceAudience;
38 import org.apache.hadoop.classification.InterfaceStability;
39 import org.apache.hadoop.conf.Configured;
40 import org.apache.hadoop.util.StringUtils;
41 import org.apache.hadoop.util.Tool;
42 import org.apache.hadoop.util.ToolRunner;
43
44 import org.apache.hadoop.conf.Configuration;
45 import org.apache.hadoop.hbase.HBaseConfiguration;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.io.HFileLink;
48 import org.apache.hadoop.hbase.io.HLogLink;
49 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
51 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.FSUtils;
54 import org.apache.hadoop.hbase.util.FSTableDescriptors;
55
56
57
58
59
60
61
62
63
64
65 @InterfaceAudience.Public
66 @InterfaceStability.Evolving
67 public final class SnapshotInfo extends Configured implements Tool {
68 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
69
70
71
72
73
74
75
76
77
78 public static class SnapshotStats {
79
80 static class FileInfo {
81 private final boolean inArchive;
82 private final long size;
83
84 FileInfo(final boolean inArchive, final long size) {
85 this.inArchive = inArchive;
86 this.size = size;
87 }
88
89
90 public boolean inArchive() {
91 return this.inArchive;
92 }
93
94
95 public boolean isMissing() {
96 return this.size < 0;
97 }
98
99
100 public long getSize() {
101 return this.size;
102 }
103
104 String getStateToString() {
105 if (isMissing()) return "NOT FOUND";
106 if (inArchive()) return "archive";
107 return null;
108 }
109 }
110
111 private int hfileArchiveCount = 0;
112 private int hfilesMissing = 0;
113 private int hfilesCount = 0;
114 private int logsMissing = 0;
115 private int logsCount = 0;
116 private long hfileArchiveSize = 0;
117 private long hfileSize = 0;
118 private long logSize = 0;
119
120 private final SnapshotDescription snapshot;
121 private final Configuration conf;
122 private final FileSystem fs;
123
124 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
125 {
126 this.snapshot = snapshot;
127 this.conf = conf;
128 this.fs = fs;
129 }
130
131
132 public SnapshotDescription getSnapshotDescription() {
133 return this.snapshot;
134 }
135
136
137 public boolean isSnapshotCorrupted() {
138 return hfilesMissing > 0 || logsMissing > 0;
139 }
140
141
142 public int getStoreFilesCount() {
143 return hfilesCount + hfileArchiveCount;
144 }
145
146
147 public int getArchivedStoreFilesCount() {
148 return hfileArchiveCount;
149 }
150
151
152 public int getLogsCount() {
153 return logsCount;
154 }
155
156
157 public int getMissingStoreFilesCount() {
158 return hfilesMissing;
159 }
160
161
162 public int getMissingLogsCount() {
163 return logsMissing;
164 }
165
166
167 public long getStoreFilesSize() {
168 return hfileSize + hfileArchiveSize;
169 }
170
171
172 public long getSharedStoreFilesSize() {
173 return hfileSize;
174 }
175
176
177 public long getArchivedStoreFileSize() {
178 return hfileArchiveSize;
179 }
180
181
182 public float getSharedStoreFilePercentage() {
183 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
184 }
185
186
187 public long getLogsSize() {
188 return logSize;
189 }
190
191
192
193
194
195
196
197
198 FileInfo addStoreFile(final String region, final String family, final String hfile)
199 throws IOException {
200 String table = this.snapshot.getTable();
201 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
202 boolean inArchive = false;
203 long size = -1;
204 try {
205 if ((inArchive = fs.exists(link.getArchivePath()))) {
206 size = fs.getFileStatus(link.getArchivePath()).getLen();
207 hfileArchiveSize += size;
208 hfileArchiveCount++;
209 } else {
210 size = link.getFileStatus(fs).getLen();
211 hfileSize += size;
212 hfilesCount++;
213 }
214 } catch (FileNotFoundException e) {
215 hfilesMissing++;
216 }
217 return new FileInfo(inArchive, size);
218 }
219
220
221
222
223
224
225
226 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
227 Path rootDir = FSUtils.getRootDir(conf);
228 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
229 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
230 long size = fs.getFileStatus(path).getLen();
231 logSize += size;
232 logsCount++;
233 return new FileInfo(true, size);
234 }
235
236
237
238
239
240
241
242 FileInfo addLogFile(final String server, final String logfile) throws IOException {
243 HLogLink logLink = new HLogLink(conf, server, logfile);
244 long size = -1;
245 try {
246 size = logLink.getFileStatus(fs).getLen();
247 logSize += size;
248 logsCount++;
249 } catch (FileNotFoundException e) {
250 logsMissing++;
251 }
252 return new FileInfo(false, size);
253 }
254 }
255
256 private boolean printSizeInBytes = false;
257 private FileSystem fs;
258 private Path rootDir;
259
260 private HTableDescriptor snapshotTableDesc;
261 private SnapshotDescription snapshotDesc;
262 private Path snapshotDir;
263
264 @Override
265 public int run(String[] args) throws IOException, InterruptedException {
266 final Configuration conf = getConf();
267 boolean listSnapshots = false;
268 String snapshotName = null;
269 boolean showSchema = false;
270 boolean showFiles = false;
271 boolean showStats = false;
272
273
274 for (int i = 0; i < args.length; i++) {
275 String cmd = args[i];
276 try {
277 if (cmd.equals("-snapshot")) {
278 snapshotName = args[++i];
279 } else if (cmd.equals("-files")) {
280 showFiles = true;
281 showStats = true;
282 } else if (cmd.equals("-stats")) {
283 showStats = true;
284 } else if (cmd.equals("-schema")) {
285 showSchema = true;
286 } else if (cmd.equals("-remote-dir")) {
287 Path sourceDir = new Path(args[++i]);
288 URI defaultFs = sourceDir.getFileSystem(conf).getUri();
289 FSUtils.setFsDefault(conf, new Path(defaultFs));
290 FSUtils.setRootDir(conf, sourceDir);
291 } else if (cmd.equals("-list-snapshots")) {
292 listSnapshots = true;
293 } else if (cmd.equals("-size-in-bytes")) {
294 printSizeInBytes = true;
295 } else if (cmd.equals("-h") || cmd.equals("--help")) {
296 printUsageAndExit();
297 } else {
298 System.err.println("UNEXPECTED: " + cmd);
299 printUsageAndExit();
300 }
301 } catch (Exception e) {
302 printUsageAndExit();
303 }
304 }
305
306
307 if (listSnapshots) {
308 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
309 System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
310 for (SnapshotDescription desc: getSnapshotList(conf)) {
311 System.out.printf("%-20s | %20s | %s%n",
312 desc.getName(),
313 df.format(new Date(desc.getCreationTime())),
314 desc.getTable());
315 }
316 return 0;
317 }
318
319 if (snapshotName == null) {
320 System.err.println("Missing snapshot name!");
321 printUsageAndExit();
322 return 1;
323 }
324
325 rootDir = FSUtils.getRootDir(conf);
326 fs = FileSystem.get(rootDir.toUri(), conf);
327 LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
328
329
330 if (!loadSnapshotInfo(snapshotName)) {
331 System.err.println("Snapshot '" + snapshotName + "' not found!");
332 return 1;
333 }
334
335 printInfo();
336 if (showSchema) printSchema();
337 printFiles(showFiles, showStats);
338
339 return 0;
340 }
341
342
343
344
345
346
347 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
348 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
349 if (!fs.exists(snapshotDir)) {
350 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
351 return false;
352 }
353
354 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
355 snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
356 return true;
357 }
358
359
360
361
362 private void printInfo() {
363 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
364 System.out.println("Snapshot Info");
365 System.out.println("----------------------------------------");
366 System.out.println(" Name: " + snapshotDesc.getName());
367 System.out.println(" Type: " + snapshotDesc.getType());
368 System.out.println(" Table: " + snapshotDesc.getTable());
369 System.out.println(" Format: " + snapshotDesc.getVersion());
370 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
371 System.out.println();
372 }
373
374
375
376
377 private void printSchema() {
378 System.out.println("Table Descriptor");
379 System.out.println("----------------------------------------");
380 System.out.println(snapshotTableDesc.toString());
381 System.out.println();
382 }
383
384
385
386
387
388 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
389 if (showFiles) {
390 System.out.println("Snapshot Files");
391 System.out.println("----------------------------------------");
392 }
393
394
395 final String table = this.snapshotDesc.getTable();
396 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
397 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
398 new SnapshotReferenceUtil.FileVisitor() {
399 public void storeFile (final String region, final String family, final String hfile)
400 throws IOException {
401 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
402
403 if (showFiles) {
404 String state = info.getStateToString();
405 System.out.printf("%8s %s/%s/%s/%s %s%n",
406 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
407 table, region, family, hfile,
408 state == null ? "" : "(" + state + ")");
409 }
410 }
411
412 public void recoveredEdits (final String region, final String logfile)
413 throws IOException {
414 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
415
416 if (showFiles) {
417 System.out.printf("%8s recovered.edits %s on region %s%n",
418 fileSizeToString(info.getSize()), logfile, region);
419 }
420 }
421
422 public void logFile (final String server, final String logfile)
423 throws IOException {
424 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
425
426 if (showFiles) {
427 String state = info.getStateToString();
428 System.out.printf("%8s log %s on server %s %s%n",
429 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
430 logfile, server,
431 state == null ? "" : "(" + state + ")");
432 }
433 }
434 });
435
436
437 System.out.println();
438 if (stats.isSnapshotCorrupted()) {
439 System.out.println("**************************************************************");
440 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
441 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
442 System.out.println("**************************************************************");
443 }
444
445 if (showStats) {
446 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
447 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
448 fileSizeToString(stats.getStoreFilesSize()),
449 stats.getSharedStoreFilePercentage(),
450 fileSizeToString(stats.getSharedStoreFilesSize())
451 );
452 System.out.printf("%d Logs, total size %s%n",
453 stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
454 System.out.println();
455 }
456 }
457
458 private String fileSizeToString(long size) {
459 return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
460 }
461
462 private void printUsageAndExit() {
463 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
464 System.err.println(" where [options] are:");
465 System.err.println(" -h|-help Show this help and exit.");
466 System.err.println(" -remote-dir Root directory that contains the snapshots.");
467 System.err.println(" -list-snapshots List all the available snapshots and exit.");
468 System.err.println(" -snapshot NAME Snapshot to examine.");
469 System.err.println(" -files Files and logs list.");
470 System.err.println(" -stats Files and logs stats.");
471 System.err.println(" -schema Describe the snapshotted table.");
472 System.err.println();
473 System.err.println("Examples:");
474 System.err.println(" hbase " + getClass() + " \\");
475 System.err.println(" -snapshot MySnapshot -files");
476 System.exit(1);
477 }
478
479
480
481
482
483
484
485 public static SnapshotStats getSnapshotStats(final Configuration conf,
486 final SnapshotDescription snapshot) throws IOException {
487 Path rootDir = FSUtils.getRootDir(conf);
488 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
489 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
490 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
491 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
492 new SnapshotReferenceUtil.FileVisitor() {
493 public void storeFile (final String region, final String family, final String hfile)
494 throws IOException {
495 stats.addStoreFile(region, family, hfile);
496 }
497
498 public void recoveredEdits (final String region, final String logfile) throws IOException {
499 stats.addRecoveredEdits(region, logfile);
500 }
501
502 public void logFile (final String server, final String logfile) throws IOException {
503 stats.addLogFile(server, logfile);
504 }
505 });
506 return stats;
507 }
508
509
510
511
512
513
514 public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
515 throws IOException {
516 Path rootDir = FSUtils.getRootDir(conf);
517 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
518 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
519 FileStatus[] snapshots = fs.listStatus(snapshotDir,
520 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
521 List<SnapshotDescription> snapshotLists =
522 new ArrayList<SnapshotDescription>(snapshots.length);
523 for (FileStatus snapshotDirStat: snapshots) {
524 snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
525 }
526 return snapshotLists;
527 }
528
529
530
531
532
533
534
535
536 static int innerMain(final String [] args) throws Exception {
537 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
538 }
539
540 public static void main(String[] args) throws Exception {
541 System.exit(innerMain(args));
542 }
543 }