1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.text.SimpleDateFormat;
24 import java.util.Date;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.classification.InterfaceAudience;
32 import org.apache.hadoop.classification.InterfaceStability;
33 import org.apache.hadoop.conf.Configured;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
36 import org.apache.hadoop.util.StringUtils;
37 import org.apache.hadoop.util.Tool;
38 import org.apache.hadoop.util.ToolRunner;
39
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.io.HFileLink;
44 import org.apache.hadoop.hbase.io.HLogLink;
45 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46 import org.apache.hadoop.hbase.util.FSUtils;
47 import org.apache.hadoop.hbase.util.FSTableDescriptors;
48
49
50
51
52
53
54
55
56
57
58 @InterfaceAudience.Public
59 @InterfaceStability.Evolving
60 public final class SnapshotInfo extends Configured implements Tool {
61 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
62
63
64
65
66
67
68
69
70
71 public static class SnapshotStats {
72
73 static class FileInfo {
74 private final boolean inArchive;
75 private final long size;
76
77 FileInfo(final boolean inArchive, final long size) {
78 this.inArchive = inArchive;
79 this.size = size;
80 }
81
82
83 public boolean inArchive() {
84 return this.inArchive;
85 }
86
87
88 public boolean isMissing() {
89 return this.size < 0;
90 }
91
92
93 public long getSize() {
94 return this.size;
95 }
96 }
97
98 private int hfileArchiveCount = 0;
99 private int hfilesMissing = 0;
100 private int hfilesCount = 0;
101 private int logsMissing = 0;
102 private int logsCount = 0;
103 private long hfileArchiveSize = 0;
104 private long hfileSize = 0;
105 private long logSize = 0;
106
107 private final SnapshotDescription snapshot;
108 private final TableName snapshotTable;
109 private final Configuration conf;
110 private final FileSystem fs;
111
112 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
113 {
114 this.snapshot = snapshot;
115 this.snapshotTable = TableName.valueOf(snapshot.getTable());
116 this.conf = conf;
117 this.fs = fs;
118 }
119
120
121 public SnapshotDescription getSnapshotDescription() {
122 return this.snapshot;
123 }
124
125
126 public boolean isSnapshotCorrupted() {
127 return hfilesMissing > 0 || logsMissing > 0;
128 }
129
130
131 public int getStoreFilesCount() {
132 return hfilesCount + hfileArchiveCount;
133 }
134
135
136 public int getArchivedStoreFilesCount() {
137 return hfileArchiveCount;
138 }
139
140
141 public int getLogsCount() {
142 return logsCount;
143 }
144
145
146 public int getMissingStoreFilesCount() {
147 return hfilesMissing;
148 }
149
150
151 public int getMissingLogsCount() {
152 return logsMissing;
153 }
154
155
156 public long getStoreFilesSize() {
157 return hfileSize + hfileArchiveSize;
158 }
159
160
161 public long getSharedStoreFilesSize() {
162 return hfileSize;
163 }
164
165
166 public long getArchivedStoreFileSize() {
167 return hfileArchiveSize;
168 }
169
170
171 public float getSharedStoreFilePercentage() {
172 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
173 }
174
175
176 public long getLogsSize() {
177 return logSize;
178 }
179
180
181
182
183
184
185
186
187 FileInfo addStoreFile(final String region, final String family, final String hfile)
188 throws IOException {
189 TableName table = snapshotTable;
190 Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
191 HFileLink link = new HFileLink(conf, path);
192 boolean inArchive = false;
193 long size = -1;
194 try {
195 if ((inArchive = fs.exists(link.getArchivePath()))) {
196 size = fs.getFileStatus(link.getArchivePath()).getLen();
197 hfileArchiveSize += size;
198 hfileArchiveCount++;
199 } else {
200 size = link.getFileStatus(fs).getLen();
201 hfileSize += size;
202 hfilesCount++;
203 }
204 } catch (FileNotFoundException e) {
205 hfilesMissing++;
206 }
207 return new FileInfo(inArchive, size);
208 }
209
210
211
212
213
214
215
216 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
217 Path rootDir = FSUtils.getRootDir(conf);
218 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
219 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
220 long size = fs.getFileStatus(path).getLen();
221 logSize += size;
222 logsCount++;
223 return new FileInfo(true, size);
224 }
225
226
227
228
229
230
231
232 FileInfo addLogFile(final String server, final String logfile) throws IOException {
233 HLogLink logLink = new HLogLink(conf, server, logfile);
234 long size = -1;
235 try {
236 size = logLink.getFileStatus(fs).getLen();
237 logSize += size;
238 logsCount++;
239 } catch (FileNotFoundException e) {
240 logsMissing++;
241 }
242 return new FileInfo(false, size);
243 }
244 }
245
246 private FileSystem fs;
247 private Path rootDir;
248
249 private HTableDescriptor snapshotTableDesc;
250 private SnapshotDescription snapshotDesc;
251 private Path snapshotDir;
252
253 @Override
254 public int run(String[] args) throws IOException, InterruptedException {
255 String snapshotName = null;
256 boolean showSchema = false;
257 boolean showFiles = false;
258 boolean showStats = false;
259
260
261 for (int i = 0; i < args.length; i++) {
262 String cmd = args[i];
263 try {
264 if (cmd.equals("-snapshot")) {
265 snapshotName = args[++i];
266 } else if (cmd.equals("-files")) {
267 showFiles = true;
268 } else if (cmd.equals("-stats")) {
269 showStats = true;
270 } else if (cmd.equals("-schema")) {
271 showSchema = true;
272 } else if (cmd.equals("-h") || cmd.equals("--help")) {
273 printUsageAndExit();
274 } else {
275 System.err.println("UNEXPECTED: " + cmd);
276 printUsageAndExit();
277 }
278 } catch (Exception e) {
279 printUsageAndExit();
280 }
281 }
282
283 if (snapshotName == null) {
284 System.err.println("Missing snapshot name!");
285 printUsageAndExit();
286 return 1;
287 }
288
289 Configuration conf = getConf();
290 fs = FileSystem.get(conf);
291 rootDir = FSUtils.getRootDir(conf);
292
293
294 if (!loadSnapshotInfo(snapshotName)) {
295 System.err.println("Snapshot '" + snapshotName + "' not found!");
296 return 1;
297 }
298
299 printInfo();
300 if (showSchema) printSchema();
301 if (showFiles || showStats) printFiles(showFiles);
302
303 return 0;
304 }
305
306
307
308
309
310
311 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
312 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
313 if (!fs.exists(snapshotDir)) {
314 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
315 return false;
316 }
317
318 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
319 snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
320 return true;
321 }
322
323
324
325
326 private void printInfo() {
327 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
328 System.out.println("Snapshot Info");
329 System.out.println("----------------------------------------");
330 System.out.println(" Name: " + snapshotDesc.getName());
331 System.out.println(" Type: " + snapshotDesc.getType());
332 System.out.println(" Table: " + snapshotTableDesc.getTableName().getNameAsString());
333 System.out.println(" Format: " + snapshotDesc.getVersion());
334 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
335 System.out.println();
336 }
337
338
339
340
341 private void printSchema() {
342 System.out.println("Table Descriptor");
343 System.out.println("----------------------------------------");
344 System.out.println(snapshotTableDesc.toString());
345 System.out.println();
346 }
347
348
349
350
351
352 private void printFiles(final boolean showFiles) throws IOException {
353 if (showFiles) {
354 System.out.println("Snapshot Files");
355 System.out.println("----------------------------------------");
356 }
357
358
359 final String table = snapshotTableDesc.getTableName().getNameAsString();
360 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
361 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
362 new SnapshotReferenceUtil.FileVisitor() {
363 public void storeFile (final String region, final String family, final String hfile)
364 throws IOException {
365 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
366
367 if (showFiles) {
368 System.out.printf("%8s %s/%s/%s/%s %s%n",
369 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
370 table, region, family, hfile,
371 (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
372 }
373 }
374
375 public void recoveredEdits (final String region, final String logfile)
376 throws IOException {
377 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
378
379 if (showFiles) {
380 System.out.printf("%8s recovered.edits %s on region %s%n",
381 StringUtils.humanReadableInt(info.getSize()), logfile, region);
382 }
383 }
384
385 public void logFile (final String server, final String logfile)
386 throws IOException {
387 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
388
389 if (showFiles) {
390 System.out.printf("%8s log %s on server %s %s%n",
391 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
392 logfile, server,
393 (info.isMissing() ? "(NOT FOUND)" : ""));
394 }
395 }
396 });
397
398
399 System.out.println();
400 if (stats.isSnapshotCorrupted()) {
401 System.out.println("**************************************************************");
402 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
403 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
404 System.out.println("**************************************************************");
405 }
406
407 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
408 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
409 StringUtils.humanReadableInt(stats.getStoreFilesSize()),
410 stats.getSharedStoreFilePercentage(),
411 StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
412 );
413 System.out.printf("%d Logs, total size %s%n",
414 stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
415 System.out.println();
416 }
417
418 private void printUsageAndExit() {
419 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
420 System.err.println(" where [options] are:");
421 System.err.println(" -h|-help Show this help and exit.");
422 System.err.println(" -snapshot NAME Snapshot to examine.");
423 System.err.println(" -files Files and logs list.");
424 System.err.println(" -stats Files and logs stats.");
425 System.err.println(" -schema Describe the snapshotted table.");
426 System.err.println();
427 System.err.println("Examples:");
428 System.err.println(" hbase " + getClass() + " \\");
429 System.err.println(" -snapshot MySnapshot -files");
430 System.exit(1);
431 }
432
433
434
435
436
437
438
439 public static SnapshotStats getSnapshotStats(final Configuration conf,
440 final SnapshotDescription snapshot) throws IOException {
441 Path rootDir = FSUtils.getRootDir(conf);
442 FileSystem fs = FileSystem.get(conf);
443 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
444 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
445 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
446 new SnapshotReferenceUtil.FileVisitor() {
447 public void storeFile (final String region, final String family, final String hfile)
448 throws IOException {
449 stats.addStoreFile(region, family, hfile);
450 }
451
452 public void recoveredEdits (final String region, final String logfile) throws IOException {
453 stats.addRecoveredEdits(region, logfile);
454 }
455
456 public void logFile (final String server, final String logfile) throws IOException {
457 stats.addLogFile(server, logfile);
458 }
459 });
460 return stats;
461 }
462
463
464
465
466
467
468
469
470 static int innerMain(final String [] args) throws Exception {
471 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
472 }
473
474 public static void main(String[] args) throws Exception {
475 System.exit(innerMain(args));
476 }
477 }