1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.net.URI;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Date;
27 import java.util.List;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.classification.InterfaceAudience;
36 import org.apache.hadoop.classification.InterfaceStability;
37 import org.apache.hadoop.conf.Configured;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.util.StringUtils;
41 import org.apache.hadoop.util.Tool;
42 import org.apache.hadoop.util.ToolRunner;
43
44 import org.apache.hadoop.conf.Configuration;
45 import org.apache.hadoop.hbase.HBaseConfiguration;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.io.HFileLink;
48 import org.apache.hadoop.hbase.io.HLogLink;
49 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.FSTableDescriptors;
52
53
54
55
56
57
58
59
60
61
62 @InterfaceAudience.Public
63 @InterfaceStability.Evolving
64 public final class SnapshotInfo extends Configured implements Tool {
65 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
66
67
68
69
70
71
72
73
74
75 public static class SnapshotStats {
76
77 static class FileInfo {
78 private final boolean inArchive;
79 private final long size;
80
81 FileInfo(final boolean inArchive, final long size) {
82 this.inArchive = inArchive;
83 this.size = size;
84 }
85
86
87 public boolean inArchive() {
88 return this.inArchive;
89 }
90
91
92 public boolean isMissing() {
93 return this.size < 0;
94 }
95
96
97 public long getSize() {
98 return this.size;
99 }
100 }
101
102 private int hfileArchiveCount = 0;
103 private int hfilesMissing = 0;
104 private int hfilesCount = 0;
105 private int logsMissing = 0;
106 private int logsCount = 0;
107 private long hfileArchiveSize = 0;
108 private long hfileSize = 0;
109 private long logSize = 0;
110
111 private final SnapshotDescription snapshot;
112 private final TableName snapshotTable;
113 private final Configuration conf;
114 private final FileSystem fs;
115
116 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
117 {
118 this.snapshot = snapshot;
119 this.snapshotTable = TableName.valueOf(snapshot.getTable());
120 this.conf = conf;
121 this.fs = fs;
122 }
123
124
125 public SnapshotDescription getSnapshotDescription() {
126 return this.snapshot;
127 }
128
129
130 public boolean isSnapshotCorrupted() {
131 return hfilesMissing > 0 || logsMissing > 0;
132 }
133
134
135 public int getStoreFilesCount() {
136 return hfilesCount + hfileArchiveCount;
137 }
138
139
140 public int getArchivedStoreFilesCount() {
141 return hfileArchiveCount;
142 }
143
144
145 public int getLogsCount() {
146 return logsCount;
147 }
148
149
150 public int getMissingStoreFilesCount() {
151 return hfilesMissing;
152 }
153
154
155 public int getMissingLogsCount() {
156 return logsMissing;
157 }
158
159
160 public long getStoreFilesSize() {
161 return hfileSize + hfileArchiveSize;
162 }
163
164
165 public long getSharedStoreFilesSize() {
166 return hfileSize;
167 }
168
169
170 public long getArchivedStoreFileSize() {
171 return hfileArchiveSize;
172 }
173
174
175 public float getSharedStoreFilePercentage() {
176 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
177 }
178
179
180 public long getLogsSize() {
181 return logSize;
182 }
183
184
185
186
187
188
189
190
191 FileInfo addStoreFile(final String region, final String family, final String hfile)
192 throws IOException {
193 TableName table = snapshotTable;
194 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
195 boolean inArchive = false;
196 long size = -1;
197 try {
198 if ((inArchive = fs.exists(link.getArchivePath()))) {
199 size = fs.getFileStatus(link.getArchivePath()).getLen();
200 hfileArchiveSize += size;
201 hfileArchiveCount++;
202 } else {
203 size = link.getFileStatus(fs).getLen();
204 hfileSize += size;
205 hfilesCount++;
206 }
207 } catch (FileNotFoundException e) {
208 hfilesMissing++;
209 }
210 return new FileInfo(inArchive, size);
211 }
212
213
214
215
216
217
218
219 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
220 Path rootDir = FSUtils.getRootDir(conf);
221 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
222 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
223 long size = fs.getFileStatus(path).getLen();
224 logSize += size;
225 logsCount++;
226 return new FileInfo(true, size);
227 }
228
229
230
231
232
233
234
235 FileInfo addLogFile(final String server, final String logfile) throws IOException {
236 HLogLink logLink = new HLogLink(conf, server, logfile);
237 long size = -1;
238 try {
239 size = logLink.getFileStatus(fs).getLen();
240 logSize += size;
241 logsCount++;
242 } catch (FileNotFoundException e) {
243 logsMissing++;
244 }
245 return new FileInfo(false, size);
246 }
247 }
248
249 private FileSystem fs;
250 private Path rootDir;
251
252 private HTableDescriptor snapshotTableDesc;
253 private SnapshotDescription snapshotDesc;
254 private Path snapshotDir;
255
256 @Override
257 public int run(String[] args) throws IOException, InterruptedException {
258 final Configuration conf = getConf();
259 boolean listSnapshots = false;
260 String snapshotName = null;
261 boolean showSchema = false;
262 boolean showFiles = false;
263 boolean showStats = false;
264
265
266 for (int i = 0; i < args.length; i++) {
267 String cmd = args[i];
268 try {
269 if (cmd.equals("-snapshot")) {
270 snapshotName = args[++i];
271 } else if (cmd.equals("-files")) {
272 showFiles = true;
273 showStats = true;
274 } else if (cmd.equals("-stats")) {
275 showStats = true;
276 } else if (cmd.equals("-schema")) {
277 showSchema = true;
278 } else if (cmd.equals("-remote-dir")) {
279 Path sourceDir = new Path(args[++i]);
280 URI defaultFs = sourceDir.getFileSystem(conf).getUri();
281 FSUtils.setFsDefault(conf, new Path(defaultFs));
282 FSUtils.setRootDir(conf, sourceDir);
283 } else if (cmd.equals("-list-snapshots")) {
284 listSnapshots = true;
285 } else if (cmd.equals("-h") || cmd.equals("--help")) {
286 printUsageAndExit();
287 } else {
288 System.err.println("UNEXPECTED: " + cmd);
289 printUsageAndExit();
290 }
291 } catch (Exception e) {
292 printUsageAndExit();
293 }
294 }
295
296
297 if (listSnapshots) {
298 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
299 System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
300 for (SnapshotDescription desc: getSnapshotList(conf)) {
301 System.out.printf("%-20s | %20s | %s%n",
302 desc.getName(),
303 df.format(new Date(desc.getCreationTime())),
304 desc.getTable());
305 }
306 return 0;
307 }
308
309 if (snapshotName == null) {
310 System.err.println("Missing snapshot name!");
311 printUsageAndExit();
312 return 1;
313 }
314
315 rootDir = FSUtils.getRootDir(conf);
316 fs = FileSystem.get(rootDir.toUri(), conf);
317 LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
318
319
320 if (!loadSnapshotInfo(snapshotName)) {
321 System.err.println("Snapshot '" + snapshotName + "' not found!");
322 return 1;
323 }
324
325 printInfo();
326 if (showSchema) printSchema();
327 printFiles(showFiles, showStats);
328
329 return 0;
330 }
331
332
333
334
335
336
337 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
338 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
339 if (!fs.exists(snapshotDir)) {
340 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
341 return false;
342 }
343
344 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
345 snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
346 return true;
347 }
348
349
350
351
352 private void printInfo() {
353 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
354 System.out.println("Snapshot Info");
355 System.out.println("----------------------------------------");
356 System.out.println(" Name: " + snapshotDesc.getName());
357 System.out.println(" Type: " + snapshotDesc.getType());
358 System.out.println(" Table: " + snapshotTableDesc.getTableName().getNameAsString());
359 System.out.println(" Format: " + snapshotDesc.getVersion());
360 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
361 System.out.println();
362 }
363
364
365
366
367 private void printSchema() {
368 System.out.println("Table Descriptor");
369 System.out.println("----------------------------------------");
370 System.out.println(snapshotTableDesc.toString());
371 System.out.println();
372 }
373
374
375
376
377
378 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
379 if (showFiles) {
380 System.out.println("Snapshot Files");
381 System.out.println("----------------------------------------");
382 }
383
384
385 final String table = snapshotTableDesc.getTableName().getNameAsString();
386 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
387 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
388 new SnapshotReferenceUtil.FileVisitor() {
389 public void storeFile (final String region, final String family, final String hfile)
390 throws IOException {
391 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
392
393 if (showFiles) {
394 System.out.printf("%8s %s/%s/%s/%s %s%n",
395 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
396 table, region, family, hfile,
397 (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
398 }
399 }
400
401 public void recoveredEdits (final String region, final String logfile)
402 throws IOException {
403 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
404
405 if (showFiles) {
406 System.out.printf("%8s recovered.edits %s on region %s%n",
407 StringUtils.humanReadableInt(info.getSize()), logfile, region);
408 }
409 }
410
411 public void logFile (final String server, final String logfile)
412 throws IOException {
413 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
414
415 if (showFiles) {
416 System.out.printf("%8s log %s on server %s %s%n",
417 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
418 logfile, server,
419 (info.isMissing() ? "(NOT FOUND)" : ""));
420 }
421 }
422 });
423
424
425 System.out.println();
426 if (stats.isSnapshotCorrupted()) {
427 System.out.println("**************************************************************");
428 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
429 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
430 System.out.println("**************************************************************");
431 }
432
433 if (showStats) {
434 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
435 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
436 StringUtils.humanReadableInt(stats.getStoreFilesSize()),
437 stats.getSharedStoreFilePercentage(),
438 StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
439 );
440 System.out.printf("%d Logs, total size %s%n",
441 stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
442 System.out.println();
443 }
444 }
445
446 private void printUsageAndExit() {
447 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
448 System.err.println(" where [options] are:");
449 System.err.println(" -h|-help Show this help and exit.");
450 System.err.println(" -remote-dir Root directory that contains the snapshots.");
451 System.err.println(" -list-snapshots List all the available snapshots and exit.");
452 System.err.println(" -snapshot NAME Snapshot to examine.");
453 System.err.println(" -files Files and logs list.");
454 System.err.println(" -stats Files and logs stats.");
455 System.err.println(" -schema Describe the snapshotted table.");
456 System.err.println();
457 System.err.println("Examples:");
458 System.err.println(" hbase " + getClass() + " \\");
459 System.err.println(" -snapshot MySnapshot -files");
460 System.exit(1);
461 }
462
463
464
465
466
467
468
469 public static SnapshotStats getSnapshotStats(final Configuration conf,
470 final SnapshotDescription snapshot) throws IOException {
471 Path rootDir = FSUtils.getRootDir(conf);
472 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
473 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
474 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
475 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
476 new SnapshotReferenceUtil.FileVisitor() {
477 public void storeFile (final String region, final String family, final String hfile)
478 throws IOException {
479 stats.addStoreFile(region, family, hfile);
480 }
481
482 public void recoveredEdits (final String region, final String logfile) throws IOException {
483 stats.addRecoveredEdits(region, logfile);
484 }
485
486 public void logFile (final String server, final String logfile) throws IOException {
487 stats.addLogFile(server, logfile);
488 }
489 });
490 return stats;
491 }
492
493
494
495
496
497
498 public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
499 throws IOException {
500 Path rootDir = FSUtils.getRootDir(conf);
501 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
502 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
503 FileStatus[] snapshots = fs.listStatus(snapshotDir,
504 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
505 List<SnapshotDescription> snapshotLists =
506 new ArrayList<SnapshotDescription>(snapshots.length);
507 for (FileStatus snapshotDirStat: snapshots) {
508 snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
509 }
510 return snapshotLists;
511 }
512
513
514
515
516
517
518
519
520 static int innerMain(final String [] args) throws Exception {
521 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
522 }
523
524 public static void main(String[] args) throws Exception {
525 System.exit(innerMain(args));
526 }
527 }