1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.net.URI;
24 import java.text.SimpleDateFormat;
25 import java.util.concurrent.atomic.AtomicInteger;
26 import java.util.concurrent.atomic.AtomicLong;
27 import java.util.ArrayList;
28 import java.util.Date;
29 import java.util.List;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.classification.InterfaceAudience;
38 import org.apache.hadoop.classification.InterfaceStability;
39 import org.apache.hadoop.conf.Configured;
40 import org.apache.hadoop.util.StringUtils;
41 import org.apache.hadoop.util.Tool;
42 import org.apache.hadoop.util.ToolRunner;
43
44 import org.apache.hadoop.conf.Configuration;
45 import org.apache.hadoop.hbase.HBaseConfiguration;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.io.HFileLink;
48 import org.apache.hadoop.hbase.io.HLogLink;
49 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
51 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.FSUtils;
54 import org.apache.hadoop.hbase.util.FSTableDescriptors;
55
56
57
58
59
60
61
62
63
64
65 @InterfaceAudience.Public
66 @InterfaceStability.Evolving
67 public final class SnapshotInfo extends Configured implements Tool {
68 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
69
70
71
72
73
74
75
76
77
78 public static class SnapshotStats {
79
80 static class FileInfo {
81 private final boolean inArchive;
82 private final long size;
83
84 FileInfo(final boolean inArchive, final long size) {
85 this.inArchive = inArchive;
86 this.size = size;
87 }
88
89
90 public boolean inArchive() {
91 return this.inArchive;
92 }
93
94
95 public boolean isMissing() {
96 return this.size < 0;
97 }
98
99
100 public long getSize() {
101 return this.size;
102 }
103 }
104
105 private int hfileArchiveCount = 0;
106 private int hfilesMissing = 0;
107 private int hfilesCount = 0;
108 private int logsMissing = 0;
109 private int logsCount = 0;
110 private long hfileArchiveSize = 0;
111 private long hfileSize = 0;
112 private long logSize = 0;
113
114 private final SnapshotDescription snapshot;
115 private final Configuration conf;
116 private final FileSystem fs;
117
118 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
119 {
120 this.snapshot = snapshot;
121 this.conf = conf;
122 this.fs = fs;
123 }
124
125
126 public SnapshotDescription getSnapshotDescription() {
127 return this.snapshot;
128 }
129
130
131 public boolean isSnapshotCorrupted() {
132 return hfilesMissing > 0 || logsMissing > 0;
133 }
134
135
136 public int getStoreFilesCount() {
137 return hfilesCount + hfileArchiveCount;
138 }
139
140
141 public int getArchivedStoreFilesCount() {
142 return hfileArchiveCount;
143 }
144
145
146 public int getLogsCount() {
147 return logsCount;
148 }
149
150
151 public int getMissingStoreFilesCount() {
152 return hfilesMissing;
153 }
154
155
156 public int getMissingLogsCount() {
157 return logsMissing;
158 }
159
160
161 public long getStoreFilesSize() {
162 return hfileSize + hfileArchiveSize;
163 }
164
165
166 public long getSharedStoreFilesSize() {
167 return hfileSize;
168 }
169
170
171 public long getArchivedStoreFileSize() {
172 return hfileArchiveSize;
173 }
174
175
176 public float getSharedStoreFilePercentage() {
177 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
178 }
179
180
181 public long getLogsSize() {
182 return logSize;
183 }
184
185
186
187
188
189
190
191
192 FileInfo addStoreFile(final String region, final String family, final String hfile)
193 throws IOException {
194 String table = this.snapshot.getTable();
195 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
196 boolean inArchive = false;
197 long size = -1;
198 try {
199 if ((inArchive = fs.exists(link.getArchivePath()))) {
200 size = fs.getFileStatus(link.getArchivePath()).getLen();
201 hfileArchiveSize += size;
202 hfileArchiveCount++;
203 } else {
204 size = link.getFileStatus(fs).getLen();
205 hfileSize += size;
206 hfilesCount++;
207 }
208 } catch (FileNotFoundException e) {
209 hfilesMissing++;
210 }
211 return new FileInfo(inArchive, size);
212 }
213
214
215
216
217
218
219
220 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
221 Path rootDir = FSUtils.getRootDir(conf);
222 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
223 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
224 long size = fs.getFileStatus(path).getLen();
225 logSize += size;
226 logsCount++;
227 return new FileInfo(true, size);
228 }
229
230
231
232
233
234
235
236 FileInfo addLogFile(final String server, final String logfile) throws IOException {
237 HLogLink logLink = new HLogLink(conf, server, logfile);
238 long size = -1;
239 try {
240 size = logLink.getFileStatus(fs).getLen();
241 logSize += size;
242 logsCount++;
243 } catch (FileNotFoundException e) {
244 logsMissing++;
245 }
246 return new FileInfo(false, size);
247 }
248 }
249
250 private FileSystem fs;
251 private Path rootDir;
252
253 private HTableDescriptor snapshotTableDesc;
254 private SnapshotDescription snapshotDesc;
255 private Path snapshotDir;
256
257 @Override
258 public int run(String[] args) throws IOException, InterruptedException {
259 final Configuration conf = getConf();
260 boolean listSnapshots = false;
261 String snapshotName = null;
262 boolean showSchema = false;
263 boolean showFiles = false;
264 boolean showStats = false;
265
266
267 for (int i = 0; i < args.length; i++) {
268 String cmd = args[i];
269 try {
270 if (cmd.equals("-snapshot")) {
271 snapshotName = args[++i];
272 } else if (cmd.equals("-files")) {
273 showFiles = true;
274 showStats = true;
275 } else if (cmd.equals("-stats")) {
276 showStats = true;
277 } else if (cmd.equals("-schema")) {
278 showSchema = true;
279 } else if (cmd.equals("-remote-dir")) {
280 Path sourceDir = new Path(args[++i]);
281 URI defaultFs = sourceDir.getFileSystem(conf).getUri();
282 FSUtils.setFsDefault(conf, new Path(defaultFs));
283 FSUtils.setRootDir(conf, sourceDir);
284 } else if (cmd.equals("-list-snapshots")) {
285 listSnapshots = true;
286 } else if (cmd.equals("-h") || cmd.equals("--help")) {
287 printUsageAndExit();
288 } else {
289 System.err.println("UNEXPECTED: " + cmd);
290 printUsageAndExit();
291 }
292 } catch (Exception e) {
293 printUsageAndExit();
294 }
295 }
296
297
298 if (listSnapshots) {
299 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
300 System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
301 for (SnapshotDescription desc: getSnapshotList(conf)) {
302 System.out.printf("%-20s | %20s | %s%n",
303 desc.getName(),
304 df.format(new Date(desc.getCreationTime())),
305 desc.getTable());
306 }
307 return 0;
308 }
309
310 if (snapshotName == null) {
311 System.err.println("Missing snapshot name!");
312 printUsageAndExit();
313 return 1;
314 }
315
316 rootDir = FSUtils.getRootDir(conf);
317 fs = FileSystem.get(rootDir.toUri(), conf);
318 LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
319
320
321 if (!loadSnapshotInfo(snapshotName)) {
322 System.err.println("Snapshot '" + snapshotName + "' not found!");
323 return 1;
324 }
325
326 printInfo();
327 if (showSchema) printSchema();
328 printFiles(showFiles, showStats);
329
330 return 0;
331 }
332
333
334
335
336
337
338 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
339 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
340 if (!fs.exists(snapshotDir)) {
341 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
342 return false;
343 }
344
345 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
346 snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
347 return true;
348 }
349
350
351
352
353 private void printInfo() {
354 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
355 System.out.println("Snapshot Info");
356 System.out.println("----------------------------------------");
357 System.out.println(" Name: " + snapshotDesc.getName());
358 System.out.println(" Type: " + snapshotDesc.getType());
359 System.out.println(" Table: " + snapshotDesc.getTable());
360 System.out.println(" Format: " + snapshotDesc.getVersion());
361 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
362 System.out.println();
363 }
364
365
366
367
368 private void printSchema() {
369 System.out.println("Table Descriptor");
370 System.out.println("----------------------------------------");
371 System.out.println(snapshotTableDesc.toString());
372 System.out.println();
373 }
374
375
376
377
378
379 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
380 if (showFiles) {
381 System.out.println("Snapshot Files");
382 System.out.println("----------------------------------------");
383 }
384
385
386 final String table = this.snapshotDesc.getTable();
387 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
388 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
389 new SnapshotReferenceUtil.FileVisitor() {
390 public void storeFile (final String region, final String family, final String hfile)
391 throws IOException {
392 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
393
394 if (showFiles) {
395 System.out.printf("%8s %s/%s/%s/%s %s%n",
396 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
397 table, region, family, hfile,
398 (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
399 }
400 }
401
402 public void recoveredEdits (final String region, final String logfile)
403 throws IOException {
404 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
405
406 if (showFiles) {
407 System.out.printf("%8s recovered.edits %s on region %s%n",
408 StringUtils.humanReadableInt(info.getSize()), logfile, region);
409 }
410 }
411
412 public void logFile (final String server, final String logfile)
413 throws IOException {
414 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
415
416 if (showFiles) {
417 System.out.printf("%8s log %s on server %s %s%n",
418 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
419 logfile, server,
420 (info.isMissing() ? "(NOT FOUND)" : ""));
421 }
422 }
423 });
424
425
426 System.out.println();
427 if (stats.isSnapshotCorrupted()) {
428 System.out.println("**************************************************************");
429 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
430 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
431 System.out.println("**************************************************************");
432 }
433
434 if (showStats) {
435 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
436 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
437 StringUtils.humanReadableInt(stats.getStoreFilesSize()),
438 stats.getSharedStoreFilePercentage(),
439 StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
440 );
441 System.out.printf("%d Logs, total size %s%n",
442 stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
443 System.out.println();
444 }
445 }
446
447 private void printUsageAndExit() {
448 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
449 System.err.println(" where [options] are:");
450 System.err.println(" -h|-help Show this help and exit.");
451 System.err.println(" -remote-dir Root directory that contains the snapshots.");
452 System.err.println(" -list-snapshots List all the available snapshots and exit.");
453 System.err.println(" -snapshot NAME Snapshot to examine.");
454 System.err.println(" -files Files and logs list.");
455 System.err.println(" -stats Files and logs stats.");
456 System.err.println(" -schema Describe the snapshotted table.");
457 System.err.println();
458 System.err.println("Examples:");
459 System.err.println(" hbase " + getClass() + " \\");
460 System.err.println(" -snapshot MySnapshot -files");
461 System.exit(1);
462 }
463
464
465
466
467
468
469
470 public static SnapshotStats getSnapshotStats(final Configuration conf,
471 final SnapshotDescription snapshot) throws IOException {
472 Path rootDir = FSUtils.getRootDir(conf);
473 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
474 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
475 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
476 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
477 new SnapshotReferenceUtil.FileVisitor() {
478 public void storeFile (final String region, final String family, final String hfile)
479 throws IOException {
480 stats.addStoreFile(region, family, hfile);
481 }
482
483 public void recoveredEdits (final String region, final String logfile) throws IOException {
484 stats.addRecoveredEdits(region, logfile);
485 }
486
487 public void logFile (final String server, final String logfile) throws IOException {
488 stats.addLogFile(server, logfile);
489 }
490 });
491 return stats;
492 }
493
494
495
496
497
498
499 public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
500 throws IOException {
501 Path rootDir = FSUtils.getRootDir(conf);
502 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
503 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
504 FileStatus[] snapshots = fs.listStatus(snapshotDir,
505 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
506 List<SnapshotDescription> snapshotLists =
507 new ArrayList<SnapshotDescription>(snapshots.length);
508 for (FileStatus snapshotDirStat: snapshots) {
509 snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
510 }
511 return snapshotLists;
512 }
513
514
515
516
517
518
519
520
521 static int innerMain(final String [] args) throws Exception {
522 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
523 }
524
525 public static void main(String[] args) throws Exception {
526 System.exit(innerMain(args));
527 }
528 }