1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.text.SimpleDateFormat;
24 import java.util.concurrent.atomic.AtomicInteger;
25 import java.util.concurrent.atomic.AtomicLong;
26 import java.util.Date;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.classification.InterfaceAudience;
34 import org.apache.hadoop.classification.InterfaceStability;
35 import org.apache.hadoop.conf.Configured;
36 import org.apache.hadoop.util.StringUtils;
37 import org.apache.hadoop.util.Tool;
38 import org.apache.hadoop.util.ToolRunner;
39
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.io.HFileLink;
44 import org.apache.hadoop.hbase.io.HLogLink;
45 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
47 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.hbase.util.FSUtils;
50 import org.apache.hadoop.hbase.util.FSTableDescriptors;
51
52
53
54
55
56
57
58
59
60
61 @InterfaceAudience.Public
62 @InterfaceStability.Evolving
63 public final class SnapshotInfo extends Configured implements Tool {
64 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
65
66
67
68
69
70
71
72
73
74 public static class SnapshotStats {
75
76 static class FileInfo {
77 private final boolean inArchive;
78 private final long size;
79
80 FileInfo(final boolean inArchive, final long size) {
81 this.inArchive = inArchive;
82 this.size = size;
83 }
84
85
86 public boolean inArchive() {
87 return this.inArchive;
88 }
89
90
91 public boolean isMissing() {
92 return this.size < 0;
93 }
94
95
96 public long getSize() {
97 return this.size;
98 }
99 }
100
101 private int hfileArchiveCount = 0;
102 private int hfilesMissing = 0;
103 private int hfilesCount = 0;
104 private int logsMissing = 0;
105 private int logsCount = 0;
106 private long hfileArchiveSize = 0;
107 private long hfileSize = 0;
108 private long logSize = 0;
109
110 private final SnapshotDescription snapshot;
111 private final Configuration conf;
112 private final FileSystem fs;
113
114 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
115 {
116 this.snapshot = snapshot;
117 this.conf = conf;
118 this.fs = fs;
119 }
120
121
122 public SnapshotDescription getSnapshotDescription() {
123 return this.snapshot;
124 }
125
126
127 public boolean isSnapshotCorrupted() {
128 return hfilesMissing > 0 || logsMissing > 0;
129 }
130
131
132 public int getStoreFilesCount() {
133 return hfilesCount + hfileArchiveCount;
134 }
135
136
137 public int getArchivedStoreFilesCount() {
138 return hfileArchiveCount;
139 }
140
141
142 public int getLogsCount() {
143 return logsCount;
144 }
145
146
147 public int getMissingStoreFilesCount() {
148 return hfilesMissing;
149 }
150
151
152 public int getMissingLogsCount() {
153 return logsMissing;
154 }
155
156
157 public long getStoreFilesSize() {
158 return hfileSize + hfileArchiveSize;
159 }
160
161
162 public long getSharedStoreFilesSize() {
163 return hfileSize;
164 }
165
166
167 public long getArchivedStoreFileSize() {
168 return hfileArchiveSize;
169 }
170
171
172 public float getSharedStoreFilePercentage() {
173 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
174 }
175
176
177 public long getLogsSize() {
178 return logSize;
179 }
180
181
182
183
184
185
186
187
188 FileInfo addStoreFile(final String region, final String family, final String hfile)
189 throws IOException {
190 String table = this.snapshot.getTable();
191 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
192 boolean inArchive = false;
193 long size = -1;
194 try {
195 if ((inArchive = fs.exists(link.getArchivePath()))) {
196 size = fs.getFileStatus(link.getArchivePath()).getLen();
197 hfileArchiveSize += size;
198 hfileArchiveCount++;
199 } else {
200 size = link.getFileStatus(fs).getLen();
201 hfileSize += size;
202 hfilesCount++;
203 }
204 } catch (FileNotFoundException e) {
205 hfilesMissing++;
206 }
207 return new FileInfo(inArchive, size);
208 }
209
210
211
212
213
214
215
216 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
217 Path rootDir = FSUtils.getRootDir(conf);
218 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
219 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
220 long size = fs.getFileStatus(path).getLen();
221 logSize += size;
222 logsCount++;
223 return new FileInfo(true, size);
224 }
225
226
227
228
229
230
231
232 FileInfo addLogFile(final String server, final String logfile) throws IOException {
233 HLogLink logLink = new HLogLink(conf, server, logfile);
234 long size = -1;
235 try {
236 size = logLink.getFileStatus(fs).getLen();
237 logSize += size;
238 logsCount++;
239 } catch (FileNotFoundException e) {
240 logsMissing++;
241 }
242 return new FileInfo(false, size);
243 }
244 }
245
246 private FileSystem fs;
247 private Path rootDir;
248
249 private HTableDescriptor snapshotTableDesc;
250 private SnapshotDescription snapshotDesc;
251 private Path snapshotDir;
252
253 @Override
254 public int run(String[] args) throws IOException, InterruptedException {
255 String snapshotName = null;
256 boolean showSchema = false;
257 boolean showFiles = false;
258 boolean showStats = false;
259
260
261 for (int i = 0; i < args.length; i++) {
262 String cmd = args[i];
263 try {
264 if (cmd.equals("-snapshot")) {
265 snapshotName = args[++i];
266 } else if (cmd.equals("-files")) {
267 showFiles = true;
268 } else if (cmd.equals("-stats")) {
269 showStats = true;
270 } else if (cmd.equals("-schema")) {
271 showSchema = true;
272 } else if (cmd.equals("-h") || cmd.equals("--help")) {
273 printUsageAndExit();
274 } else {
275 System.err.println("UNEXPECTED: " + cmd);
276 printUsageAndExit();
277 }
278 } catch (Exception e) {
279 printUsageAndExit();
280 }
281 }
282
283 if (snapshotName == null) {
284 System.err.println("Missing snapshot name!");
285 printUsageAndExit();
286 return 1;
287 }
288
289 Configuration conf = getConf();
290 fs = FileSystem.get(conf);
291 rootDir = FSUtils.getRootDir(conf);
292
293
294 if (!loadSnapshotInfo(snapshotName)) {
295 System.err.println("Snapshot '" + snapshotName + "' not found!");
296 return 1;
297 }
298
299 printInfo();
300 if (showSchema) printSchema();
301 if (showFiles || showStats) printFiles(showFiles);
302
303 return 0;
304 }
305
306
307
308
309
310
311 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
312 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
313 if (!fs.exists(snapshotDir)) {
314 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
315 return false;
316 }
317
318 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
319 snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
320 return true;
321 }
322
323
324
325
326 private void printInfo() {
327 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
328 System.out.println("Snapshot Info");
329 System.out.println("----------------------------------------");
330 System.out.println(" Name: " + snapshotDesc.getName());
331 System.out.println(" Type: " + snapshotDesc.getType());
332 System.out.println(" Table: " + snapshotDesc.getTable());
333 System.out.println(" Format: " + snapshotDesc.getVersion());
334 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
335 System.out.println();
336 }
337
338
339
340
341 private void printSchema() {
342 System.out.println("Table Descriptor");
343 System.out.println("----------------------------------------");
344 System.out.println(snapshotTableDesc.toString());
345 System.out.println();
346 }
347
348
349
350
351
352 private void printFiles(final boolean showFiles) throws IOException {
353 if (showFiles) {
354 System.out.println("Snapshot Files");
355 System.out.println("----------------------------------------");
356 }
357
358
359 final String table = this.snapshotDesc.getTable();
360 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
361 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
362 new SnapshotReferenceUtil.FileVisitor() {
363 public void storeFile (final String region, final String family, final String hfile)
364 throws IOException {
365 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
366
367 if (showFiles) {
368 System.out.printf("%8s %s/%s/%s/%s %s%n",
369 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
370 table, region, family, hfile,
371 (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
372 }
373 }
374
375 public void recoveredEdits (final String region, final String logfile)
376 throws IOException {
377 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
378
379 if (showFiles) {
380 System.out.printf("%8s recovered.edits %s on region %s%n",
381 StringUtils.humanReadableInt(info.getSize()), logfile, region);
382 }
383 }
384
385 public void logFile (final String server, final String logfile)
386 throws IOException {
387 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
388
389 if (showFiles) {
390 System.out.printf("%8s log %s on server %s %s%n",
391 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
392 logfile, server,
393 (info.isMissing() ? "(NOT FOUND)" : ""));
394 }
395 }
396 });
397
398
399 System.out.println();
400 if (stats.isSnapshotCorrupted()) {
401 System.out.println("**************************************************************");
402 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
403 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
404 System.out.println("**************************************************************");
405 }
406
407 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
408 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
409 StringUtils.humanReadableInt(stats.getStoreFilesSize()),
410 stats.getSharedStoreFilePercentage(),
411 StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
412 );
413 System.out.printf("%d Logs, total size %s%n",
414 stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
415 System.out.println();
416 }
417
418 private void printUsageAndExit() {
419 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
420 System.err.println(" where [options] are:");
421 System.err.println(" -h|-help Show this help and exit.");
422 System.err.println(" -snapshot NAME Snapshot to examine.");
423 System.err.println(" -files Files and logs list.");
424 System.err.println(" -stats Files and logs stats.");
425 System.err.println(" -schema Describe the snapshotted table.");
426 System.err.println();
427 System.err.println("Examples:");
428 System.err.println(" hbase " + getClass() + " \\");
429 System.err.println(" -snapshot MySnapshot -files");
430 System.exit(1);
431 }
432
433
434
435
436
437
438
439 public static SnapshotStats getSnapshotStats(final Configuration conf,
440 final SnapshotDescription snapshot) throws IOException {
441 Path rootDir = FSUtils.getRootDir(conf);
442 FileSystem fs = FileSystem.get(conf);
443 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
444 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
445 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
446 new SnapshotReferenceUtil.FileVisitor() {
447 public void storeFile (final String region, final String family, final String hfile)
448 throws IOException {
449 stats.addStoreFile(region, family, hfile);
450 }
451
452 public void recoveredEdits (final String region, final String logfile) throws IOException {
453 stats.addRecoveredEdits(region, logfile);
454 }
455
456 public void logFile (final String server, final String logfile) throws IOException {
457 stats.addLogFile(server, logfile);
458 }
459 });
460 return stats;
461 }
462
463
464
465
466
467
468
469
470 static int innerMain(final String [] args) throws Exception {
471 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
472 }
473
474 public static void main(String[] args) throws Exception {
475 System.exit(innerMain(args));
476 }
477 }