1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.text.SimpleDateFormat;
24 import java.util.Date;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.classification.InterfaceAudience;
32 import org.apache.hadoop.classification.InterfaceStability;
33 import org.apache.hadoop.conf.Configured;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
36 import org.apache.hadoop.util.StringUtils;
37 import org.apache.hadoop.util.Tool;
38 import org.apache.hadoop.util.ToolRunner;
39
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.io.HFileLink;
44 import org.apache.hadoop.hbase.io.HLogLink;
45 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46 import org.apache.hadoop.hbase.util.FSUtils;
47 import org.apache.hadoop.hbase.util.FSTableDescriptors;
48
49
50
51
52
53
54
55
56
57
58 @InterfaceAudience.Public
59 @InterfaceStability.Evolving
60 public final class SnapshotInfo extends Configured implements Tool {
61 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
62
63
64
65
66
67
68
69
70
71 public static class SnapshotStats {
72
73 static class FileInfo {
74 private final boolean inArchive;
75 private final long size;
76
77 FileInfo(final boolean inArchive, final long size) {
78 this.inArchive = inArchive;
79 this.size = size;
80 }
81
82
83 public boolean inArchive() {
84 return this.inArchive;
85 }
86
87
88 public boolean isMissing() {
89 return this.size < 0;
90 }
91
92
93 public long getSize() {
94 return this.size;
95 }
96 }
97
98 private int hfileArchiveCount = 0;
99 private int hfilesMissing = 0;
100 private int hfilesCount = 0;
101 private int logsMissing = 0;
102 private int logsCount = 0;
103 private long hfileArchiveSize = 0;
104 private long hfileSize = 0;
105 private long logSize = 0;
106
107 private final SnapshotDescription snapshot;
108 private final TableName snapshotTable;
109 private final Configuration conf;
110 private final FileSystem fs;
111
112 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
113 {
114 this.snapshot = snapshot;
115 this.snapshotTable = TableName.valueOf(snapshot.getTable());
116 this.conf = conf;
117 this.fs = fs;
118 }
119
120
121 public SnapshotDescription getSnapshotDescription() {
122 return this.snapshot;
123 }
124
125
126 public boolean isSnapshotCorrupted() {
127 return hfilesMissing > 0 || logsMissing > 0;
128 }
129
130
131 public int getStoreFilesCount() {
132 return hfilesCount + hfileArchiveCount;
133 }
134
135
136 public int getArchivedStoreFilesCount() {
137 return hfileArchiveCount;
138 }
139
140
141 public int getLogsCount() {
142 return logsCount;
143 }
144
145
146 public int getMissingStoreFilesCount() {
147 return hfilesMissing;
148 }
149
150
151 public int getMissingLogsCount() {
152 return logsMissing;
153 }
154
155
156 public long getStoreFilesSize() {
157 return hfileSize + hfileArchiveSize;
158 }
159
160
161 public long getSharedStoreFilesSize() {
162 return hfileSize;
163 }
164
165
166 public long getArchivedStoreFileSize() {
167 return hfileArchiveSize;
168 }
169
170
171 public float getSharedStoreFilePercentage() {
172 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
173 }
174
175
176 public long getLogsSize() {
177 return logSize;
178 }
179
180
181
182
183
184
185
186
187 FileInfo addStoreFile(final String region, final String family, final String hfile)
188 throws IOException {
189 TableName table = snapshotTable;
190 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
191 boolean inArchive = false;
192 long size = -1;
193 try {
194 if ((inArchive = fs.exists(link.getArchivePath()))) {
195 size = fs.getFileStatus(link.getArchivePath()).getLen();
196 hfileArchiveSize += size;
197 hfileArchiveCount++;
198 } else {
199 size = link.getFileStatus(fs).getLen();
200 hfileSize += size;
201 hfilesCount++;
202 }
203 } catch (FileNotFoundException e) {
204 hfilesMissing++;
205 }
206 return new FileInfo(inArchive, size);
207 }
208
209
210
211
212
213
214
215 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
216 Path rootDir = FSUtils.getRootDir(conf);
217 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
218 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
219 long size = fs.getFileStatus(path).getLen();
220 logSize += size;
221 logsCount++;
222 return new FileInfo(true, size);
223 }
224
225
226
227
228
229
230
231 FileInfo addLogFile(final String server, final String logfile) throws IOException {
232 HLogLink logLink = new HLogLink(conf, server, logfile);
233 long size = -1;
234 try {
235 size = logLink.getFileStatus(fs).getLen();
236 logSize += size;
237 logsCount++;
238 } catch (FileNotFoundException e) {
239 logsMissing++;
240 }
241 return new FileInfo(false, size);
242 }
243 }
244
245 private FileSystem fs;
246 private Path rootDir;
247
248 private HTableDescriptor snapshotTableDesc;
249 private SnapshotDescription snapshotDesc;
250 private Path snapshotDir;
251
252 @Override
253 public int run(String[] args) throws IOException, InterruptedException {
254 String snapshotName = null;
255 boolean showSchema = false;
256 boolean showFiles = false;
257 boolean showStats = false;
258
259
260 for (int i = 0; i < args.length; i++) {
261 String cmd = args[i];
262 try {
263 if (cmd.equals("-snapshot")) {
264 snapshotName = args[++i];
265 } else if (cmd.equals("-files")) {
266 showFiles = true;
267 } else if (cmd.equals("-stats")) {
268 showStats = true;
269 } else if (cmd.equals("-schema")) {
270 showSchema = true;
271 } else if (cmd.equals("-h") || cmd.equals("--help")) {
272 printUsageAndExit();
273 } else {
274 System.err.println("UNEXPECTED: " + cmd);
275 printUsageAndExit();
276 }
277 } catch (Exception e) {
278 printUsageAndExit();
279 }
280 }
281
282 if (snapshotName == null) {
283 System.err.println("Missing snapshot name!");
284 printUsageAndExit();
285 return 1;
286 }
287
288 Configuration conf = getConf();
289 fs = FileSystem.get(conf);
290 rootDir = FSUtils.getRootDir(conf);
291
292
293 if (!loadSnapshotInfo(snapshotName)) {
294 System.err.println("Snapshot '" + snapshotName + "' not found!");
295 return 1;
296 }
297
298 printInfo();
299 if (showSchema) printSchema();
300 if (showFiles || showStats) printFiles(showFiles);
301
302 return 0;
303 }
304
305
306
307
308
309
310 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
311 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
312 if (!fs.exists(snapshotDir)) {
313 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
314 return false;
315 }
316
317 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
318 snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
319 return true;
320 }
321
322
323
324
325 private void printInfo() {
326 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
327 System.out.println("Snapshot Info");
328 System.out.println("----------------------------------------");
329 System.out.println(" Name: " + snapshotDesc.getName());
330 System.out.println(" Type: " + snapshotDesc.getType());
331 System.out.println(" Table: " + snapshotTableDesc.getTableName().getNameAsString());
332 System.out.println(" Format: " + snapshotDesc.getVersion());
333 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
334 System.out.println();
335 }
336
337
338
339
340 private void printSchema() {
341 System.out.println("Table Descriptor");
342 System.out.println("----------------------------------------");
343 System.out.println(snapshotTableDesc.toString());
344 System.out.println();
345 }
346
347
348
349
350
351 private void printFiles(final boolean showFiles) throws IOException {
352 if (showFiles) {
353 System.out.println("Snapshot Files");
354 System.out.println("----------------------------------------");
355 }
356
357
358 final String table = snapshotTableDesc.getTableName().getNameAsString();
359 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
360 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
361 new SnapshotReferenceUtil.FileVisitor() {
362 public void storeFile (final String region, final String family, final String hfile)
363 throws IOException {
364 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
365
366 if (showFiles) {
367 System.out.printf("%8s %s/%s/%s/%s %s%n",
368 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
369 table, region, family, hfile,
370 (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
371 }
372 }
373
374 public void recoveredEdits (final String region, final String logfile)
375 throws IOException {
376 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
377
378 if (showFiles) {
379 System.out.printf("%8s recovered.edits %s on region %s%n",
380 StringUtils.humanReadableInt(info.getSize()), logfile, region);
381 }
382 }
383
384 public void logFile (final String server, final String logfile)
385 throws IOException {
386 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
387
388 if (showFiles) {
389 System.out.printf("%8s log %s on server %s %s%n",
390 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
391 logfile, server,
392 (info.isMissing() ? "(NOT FOUND)" : ""));
393 }
394 }
395 });
396
397
398 System.out.println();
399 if (stats.isSnapshotCorrupted()) {
400 System.out.println("**************************************************************");
401 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
402 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
403 System.out.println("**************************************************************");
404 }
405
406 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
407 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
408 StringUtils.humanReadableInt(stats.getStoreFilesSize()),
409 stats.getSharedStoreFilePercentage(),
410 StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
411 );
412 System.out.printf("%d Logs, total size %s%n",
413 stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
414 System.out.println();
415 }
416
417 private void printUsageAndExit() {
418 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
419 System.err.println(" where [options] are:");
420 System.err.println(" -h|-help Show this help and exit.");
421 System.err.println(" -snapshot NAME Snapshot to examine.");
422 System.err.println(" -files Files and logs list.");
423 System.err.println(" -stats Files and logs stats.");
424 System.err.println(" -schema Describe the snapshotted table.");
425 System.err.println();
426 System.err.println("Examples:");
427 System.err.println(" hbase " + getClass() + " \\");
428 System.err.println(" -snapshot MySnapshot -files");
429 System.exit(1);
430 }
431
432
433
434
435
436
437
438 public static SnapshotStats getSnapshotStats(final Configuration conf,
439 final SnapshotDescription snapshot) throws IOException {
440 Path rootDir = FSUtils.getRootDir(conf);
441 FileSystem fs = FileSystem.get(conf);
442 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
443 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
444 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
445 new SnapshotReferenceUtil.FileVisitor() {
446 public void storeFile (final String region, final String family, final String hfile)
447 throws IOException {
448 stats.addStoreFile(region, family, hfile);
449 }
450
451 public void recoveredEdits (final String region, final String logfile) throws IOException {
452 stats.addRecoveredEdits(region, logfile);
453 }
454
455 public void logFile (final String server, final String logfile) throws IOException {
456 stats.addLogFile(server, logfile);
457 }
458 });
459 return stats;
460 }
461
462
463
464
465
466
467
468
469 static int innerMain(final String [] args) throws Exception {
470 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
471 }
472
473 public static void main(String[] args) throws Exception {
474 System.exit(innerMain(args));
475 }
476 }