1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.text.SimpleDateFormat;
24 import java.util.concurrent.atomic.AtomicInteger;
25 import java.util.concurrent.atomic.AtomicLong;
26 import java.util.Date;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.classification.InterfaceAudience;
34 import org.apache.hadoop.classification.InterfaceStability;
35 import org.apache.hadoop.conf.Configured;
36 import org.apache.hadoop.util.StringUtils;
37 import org.apache.hadoop.util.Tool;
38 import org.apache.hadoop.util.ToolRunner;
39
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.io.HFileLink;
44 import org.apache.hadoop.hbase.io.HLogLink;
45 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
47 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.hbase.util.FSUtils;
50 import org.apache.hadoop.hbase.util.FSTableDescriptors;
51
52
53
54
55
56
57
58
59
60
61 @InterfaceAudience.Public
62 @InterfaceStability.Evolving
63 public final class SnapshotInfo extends Configured implements Tool {
64 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
65
66
67
68
69
70
71
72
73
74 public static class SnapshotStats {
75
76 static class FileInfo {
77 private final boolean inArchive;
78 private final long size;
79
80 FileInfo(final boolean inArchive, final long size) {
81 this.inArchive = inArchive;
82 this.size = size;
83 }
84
85
86 public boolean inArchive() {
87 return this.inArchive;
88 }
89
90
91 public boolean isMissing() {
92 return this.size < 0;
93 }
94
95
96 public long getSize() {
97 return this.size;
98 }
99 }
100
101 private int hfileArchiveCount = 0;
102 private int hfilesMissing = 0;
103 private int hfilesCount = 0;
104 private int logsMissing = 0;
105 private int logsCount = 0;
106 private long hfileArchiveSize = 0;
107 private long hfileSize = 0;
108 private long logSize = 0;
109
110 private final SnapshotDescription snapshot;
111 private final Configuration conf;
112 private final FileSystem fs;
113
114 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
115 {
116 this.snapshot = snapshot;
117 this.conf = conf;
118 this.fs = fs;
119 }
120
121
122 public SnapshotDescription getSnapshotDescription() {
123 return this.snapshot;
124 }
125
126
127 public boolean isSnapshotCorrupted() {
128 return hfilesMissing > 0 || logsMissing > 0;
129 }
130
131
132 public int getStoreFilesCount() {
133 return hfilesCount + hfileArchiveCount;
134 }
135
136
137 public int getArchivedStoreFilesCount() {
138 return hfileArchiveCount;
139 }
140
141
142 public int getLogsCount() {
143 return logsCount;
144 }
145
146
147 public int getMissingStoreFilesCount() {
148 return hfilesMissing;
149 }
150
151
152 public int getMissingLogsCount() {
153 return logsMissing;
154 }
155
156
157 public long getStoreFilesSize() {
158 return hfileSize + hfileArchiveSize;
159 }
160
161
162 public long getSharedStoreFilesSize() {
163 return hfileSize;
164 }
165
166
167 public long getArchivedStoreFileSize() {
168 return hfileArchiveSize;
169 }
170
171
172 public float getSharedStoreFilePercentage() {
173 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
174 }
175
176
177 public long getLogsSize() {
178 return logSize;
179 }
180
181
182
183
184
185
186
187
188 FileInfo addStoreFile(final String region, final String family, final String hfile)
189 throws IOException {
190 String table = this.snapshot.getTable();
191 Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile));
192 HFileLink link = new HFileLink(conf, path);
193 boolean inArchive = false;
194 long size = -1;
195 try {
196 if ((inArchive = fs.exists(link.getArchivePath()))) {
197 size = fs.getFileStatus(link.getArchivePath()).getLen();
198 hfileArchiveSize += size;
199 hfileArchiveCount++;
200 } else {
201 size = link.getFileStatus(fs).getLen();
202 hfileSize += size;
203 hfilesCount++;
204 }
205 } catch (FileNotFoundException e) {
206 hfilesMissing++;
207 }
208 return new FileInfo(inArchive, size);
209 }
210
211
212
213
214
215
216
217 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
218 Path rootDir = FSUtils.getRootDir(conf);
219 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
220 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
221 long size = fs.getFileStatus(path).getLen();
222 logSize += size;
223 logsCount++;
224 return new FileInfo(true, size);
225 }
226
227
228
229
230
231
232
233 FileInfo addLogFile(final String server, final String logfile) throws IOException {
234 HLogLink logLink = new HLogLink(conf, server, logfile);
235 long size = -1;
236 try {
237 size = logLink.getFileStatus(fs).getLen();
238 logSize += size;
239 logsCount++;
240 } catch (FileNotFoundException e) {
241 logsMissing++;
242 }
243 return new FileInfo(false, size);
244 }
245 }
246
247 private FileSystem fs;
248 private Path rootDir;
249
250 private HTableDescriptor snapshotTableDesc;
251 private SnapshotDescription snapshotDesc;
252 private Path snapshotDir;
253
254 @Override
255 public int run(String[] args) throws IOException, InterruptedException {
256 String snapshotName = null;
257 boolean showSchema = false;
258 boolean showFiles = false;
259 boolean showStats = false;
260
261
262 for (int i = 0; i < args.length; i++) {
263 String cmd = args[i];
264 try {
265 if (cmd.equals("-snapshot")) {
266 snapshotName = args[++i];
267 } else if (cmd.equals("-files")) {
268 showFiles = true;
269 } else if (cmd.equals("-stats")) {
270 showStats = true;
271 } else if (cmd.equals("-schema")) {
272 showSchema = true;
273 } else if (cmd.equals("-h") || cmd.equals("--help")) {
274 printUsageAndExit();
275 } else {
276 System.err.println("UNEXPECTED: " + cmd);
277 printUsageAndExit();
278 }
279 } catch (Exception e) {
280 printUsageAndExit();
281 }
282 }
283
284 if (snapshotName == null) {
285 System.err.println("Missing snapshot name!");
286 printUsageAndExit();
287 return 1;
288 }
289
290 Configuration conf = getConf();
291 fs = FileSystem.get(conf);
292 rootDir = FSUtils.getRootDir(conf);
293
294
295 if (!loadSnapshotInfo(snapshotName)) {
296 System.err.println("Snapshot '" + snapshotName + "' not found!");
297 return 1;
298 }
299
300 printInfo();
301 if (showSchema) printSchema();
302 if (showFiles || showStats) printFiles(showFiles);
303
304 return 0;
305 }
306
307
308
309
310
311
312 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
313 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
314 if (!fs.exists(snapshotDir)) {
315 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
316 return false;
317 }
318
319 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
320 snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
321 return true;
322 }
323
324
325
326
327 private void printInfo() {
328 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
329 System.out.println("Snapshot Info");
330 System.out.println("----------------------------------------");
331 System.out.println(" Name: " + snapshotDesc.getName());
332 System.out.println(" Type: " + snapshotDesc.getType());
333 System.out.println(" Table: " + snapshotDesc.getTable());
334 System.out.println(" Format: " + snapshotDesc.getVersion());
335 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
336 System.out.println();
337 }
338
339
340
341
342 private void printSchema() {
343 System.out.println("Table Descriptor");
344 System.out.println("----------------------------------------");
345 System.out.println(snapshotTableDesc.toString());
346 System.out.println();
347 }
348
349
350
351
352
353 private void printFiles(final boolean showFiles) throws IOException {
354 if (showFiles) {
355 System.out.println("Snapshot Files");
356 System.out.println("----------------------------------------");
357 }
358
359
360 final String table = this.snapshotDesc.getTable();
361 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
362 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
363 new SnapshotReferenceUtil.FileVisitor() {
364 public void storeFile (final String region, final String family, final String hfile)
365 throws IOException {
366 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
367
368 if (showFiles) {
369 System.out.printf("%8s %s/%s/%s/%s %s%n",
370 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
371 table, region, family, hfile,
372 (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
373 }
374 }
375
376 public void recoveredEdits (final String region, final String logfile)
377 throws IOException {
378 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
379
380 if (showFiles) {
381 System.out.printf("%8s recovered.edits %s on region %s%n",
382 StringUtils.humanReadableInt(info.getSize()), logfile, region);
383 }
384 }
385
386 public void logFile (final String server, final String logfile)
387 throws IOException {
388 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
389
390 if (showFiles) {
391 System.out.printf("%8s log %s on server %s %s%n",
392 (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
393 logfile, server,
394 (info.isMissing() ? "(NOT FOUND)" : ""));
395 }
396 }
397 });
398
399
400 System.out.println();
401 if (stats.isSnapshotCorrupted()) {
402 System.out.println("**************************************************************");
403 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
404 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
405 System.out.println("**************************************************************");
406 }
407
408 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
409 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
410 StringUtils.humanReadableInt(stats.getStoreFilesSize()),
411 stats.getSharedStoreFilePercentage(),
412 StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
413 );
414 System.out.printf("%d Logs, total size %s%n",
415 stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
416 System.out.println();
417 }
418
419 private void printUsageAndExit() {
420 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
421 System.err.println(" where [options] are:");
422 System.err.println(" -h|-help Show this help and exit.");
423 System.err.println(" -snapshot NAME Snapshot to examine.");
424 System.err.println(" -files Files and logs list.");
425 System.err.println(" -stats Files and logs stats.");
426 System.err.println(" -schema Describe the snapshotted table.");
427 System.err.println();
428 System.err.println("Examples:");
429 System.err.println(" hbase " + getClass() + " \\");
430 System.err.println(" -snapshot MySnapshot -files");
431 System.exit(1);
432 }
433
434
435
436
437
438
439
440 public static SnapshotStats getSnapshotStats(final Configuration conf,
441 final SnapshotDescription snapshot) throws IOException {
442 Path rootDir = FSUtils.getRootDir(conf);
443 FileSystem fs = FileSystem.get(conf);
444 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
445 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
446 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
447 new SnapshotReferenceUtil.FileVisitor() {
448 public void storeFile (final String region, final String family, final String hfile)
449 throws IOException {
450 stats.addStoreFile(region, family, hfile);
451 }
452
453 public void recoveredEdits (final String region, final String logfile) throws IOException {
454 stats.addRecoveredEdits(region, logfile);
455 }
456
457 public void logFile (final String server, final String logfile) throws IOException {
458 stats.addLogFile(server, logfile);
459 }
460 });
461 return stats;
462 }
463
464
465
466
467
468
469
470
471 static int innerMain(final String [] args) throws Exception {
472 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
473 }
474
475 public static void main(String[] args) throws Exception {
476 System.exit(innerMain(args));
477 }
478 }