View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.text.SimpleDateFormat;
24  import java.util.Date;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.classification.InterfaceAudience;
32  import org.apache.hadoop.classification.InterfaceStability;
33  import org.apache.hadoop.conf.Configured;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
36  import org.apache.hadoop.util.StringUtils;
37  import org.apache.hadoop.util.Tool;
38  import org.apache.hadoop.util.ToolRunner;
39  
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.io.HFileLink;
44  import org.apache.hadoop.hbase.io.HLogLink;
45  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46  import org.apache.hadoop.hbase.util.FSUtils;
47  import org.apache.hadoop.hbase.util.FSTableDescriptors;
48  
49  /**
50   * Tool for dumping snapshot information.
51   * <ol>
52   * <li> Table Descriptor
53   * <li> Snapshot creation time, type, format version, ...
54   * <li> List of hfiles and hlogs
55   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
56   * </ol>
57   */
58  @InterfaceAudience.Public
59  @InterfaceStability.Evolving
60  public final class SnapshotInfo extends Configured implements Tool {
61    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
62  
63    /**
64     * Statistics about the snapshot
65     * <ol>
66     * <li> How many store files and logs are in the archive
67     * <li> How many store files and logs are shared with the table
68     * <li> Total store files and logs size and shared amount
69     * </ol>
70     */
71    public static class SnapshotStats {
72      /** Information about the file referenced by the snapshot */
73      static class FileInfo {
74        private final boolean inArchive;
75        private final long size;
76  
77        FileInfo(final boolean inArchive, final long size) {
78          this.inArchive = inArchive;
79          this.size = size;
80        }
81  
82        /** @return true if the file is in the archive */
83        public boolean inArchive() {
84          return this.inArchive;
85        }
86  
87        /** @return true if the file is missing */
88        public boolean isMissing() {
89          return this.size < 0;
90        }
91  
92        /** @return the file size */
93        public long getSize() {
94          return this.size;
95        }
96      }
97  
98      private int hfileArchiveCount = 0;
99      private int hfilesMissing = 0;
100     private int hfilesCount = 0;
101     private int logsMissing = 0;
102     private int logsCount = 0;
103     private long hfileArchiveSize = 0;
104     private long hfileSize = 0;
105     private long logSize = 0;
106 
107     private final SnapshotDescription snapshot;
108     private final TableName snapshotTable;
109     private final Configuration conf;
110     private final FileSystem fs;
111 
112     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
113     {
114       this.snapshot = snapshot;
115       this.snapshotTable = TableName.valueOf(snapshot.getTable());
116       this.conf = conf;
117       this.fs = fs;
118     }
119 
120     /** @return the snapshot descriptor */
121     public SnapshotDescription getSnapshotDescription() {
122       return this.snapshot;
123     }
124 
125     /** @return true if the snapshot is corrupted */
126     public boolean isSnapshotCorrupted() {
127       return hfilesMissing > 0 || logsMissing > 0;
128     }
129 
130     /** @return the number of available store files */
131     public int getStoreFilesCount() {
132       return hfilesCount + hfileArchiveCount;
133     }
134 
135     /** @return the number of available store files in the archive */
136     public int getArchivedStoreFilesCount() {
137       return hfileArchiveCount;
138     }
139 
140     /** @return the number of available log files */
141     public int getLogsCount() {
142       return logsCount;
143     }
144 
145     /** @return the number of missing store files */
146     public int getMissingStoreFilesCount() {
147       return hfilesMissing;
148     }
149 
150     /** @return the number of missing log files */
151     public int getMissingLogsCount() {
152       return logsMissing;
153     }
154 
155     /** @return the total size of the store files referenced by the snapshot */
156     public long getStoreFilesSize() {
157       return hfileSize + hfileArchiveSize;
158     }
159 
160     /** @return the total size of the store files shared */
161     public long getSharedStoreFilesSize() {
162       return hfileSize;
163     }
164 
165     /** @return the total size of the store files in the archive */
166     public long getArchivedStoreFileSize() {
167       return hfileArchiveSize;
168     }
169 
170     /** @return the percentage of the shared store files */
171     public float getSharedStoreFilePercentage() {
172       return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
173     }
174 
175     /** @return the total log size */
176     public long getLogsSize() {
177       return logSize;
178     }
179 
180     /**
181      * Add the specified store file to the stats
182      * @param region region encoded Name
183      * @param family family name
184      * @param hfile store file name
185      * @return the store file information
186      */
187     FileInfo addStoreFile(final String region, final String family, final String hfile)
188           throws IOException {
189       TableName table = snapshotTable;
190       HFileLink link = HFileLink.create(conf, table, region, family, hfile);
191       boolean inArchive = false;
192       long size = -1;
193       try {
194         if ((inArchive = fs.exists(link.getArchivePath()))) {
195           size = fs.getFileStatus(link.getArchivePath()).getLen();
196           hfileArchiveSize += size;
197           hfileArchiveCount++;
198         } else {
199           size = link.getFileStatus(fs).getLen();
200           hfileSize += size;
201           hfilesCount++;
202         }
203       } catch (FileNotFoundException e) {
204         hfilesMissing++;
205       }
206       return new FileInfo(inArchive, size);
207     }
208 
209     /**
210      * Add the specified recovered.edits file to the stats
211      * @param region region encoded name
212      * @param logfile log file name
213      * @return the recovered.edits information
214      */
215     FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
216       Path rootDir = FSUtils.getRootDir(conf);
217       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
218       Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
219       long size = fs.getFileStatus(path).getLen();
220       logSize += size;
221       logsCount++;
222       return new FileInfo(true, size);
223     }
224 
225     /**
226      * Add the specified log file to the stats
227      * @param server server name
228      * @param logfile log file name
229      * @return the log information
230      */
231     FileInfo addLogFile(final String server, final String logfile) throws IOException {
232       HLogLink logLink = new HLogLink(conf, server, logfile);
233       long size = -1;
234       try {
235         size = logLink.getFileStatus(fs).getLen();
236         logSize += size;
237         logsCount++;
238       } catch (FileNotFoundException e) {
239         logsMissing++;
240       }
241       return new FileInfo(false, size);
242     }
243   }
244 
245   private FileSystem fs;
246   private Path rootDir;
247 
248   private HTableDescriptor snapshotTableDesc;
249   private SnapshotDescription snapshotDesc;
250   private Path snapshotDir;
251 
252   @Override
253   public int run(String[] args) throws IOException, InterruptedException {
254     String snapshotName = null;
255     boolean showSchema = false;
256     boolean showFiles = false;
257     boolean showStats = false;
258 
259     // Process command line args
260     for (int i = 0; i < args.length; i++) {
261       String cmd = args[i];
262       try {
263         if (cmd.equals("-snapshot")) {
264           snapshotName = args[++i];
265         } else if (cmd.equals("-files")) {
266           showFiles = true;
267         } else if (cmd.equals("-stats")) {
268           showStats = true;
269         } else if (cmd.equals("-schema")) {
270           showSchema = true;
271         } else if (cmd.equals("-h") || cmd.equals("--help")) {
272           printUsageAndExit();
273         } else {
274           System.err.println("UNEXPECTED: " + cmd);
275           printUsageAndExit();
276         }
277       } catch (Exception e) {
278         printUsageAndExit();
279       }
280     }
281 
282     if (snapshotName == null) {
283       System.err.println("Missing snapshot name!");
284       printUsageAndExit();
285       return 1;
286     }
287 
288     Configuration conf = getConf();
289     fs = FileSystem.get(conf);
290     rootDir = FSUtils.getRootDir(conf);
291 
292     // Load snapshot information
293     if (!loadSnapshotInfo(snapshotName)) {
294       System.err.println("Snapshot '" + snapshotName + "' not found!");
295       return 1;
296     }
297 
298     printInfo();
299     if (showSchema) printSchema();
300     if (showFiles || showStats) printFiles(showFiles);
301 
302     return 0;
303   }
304 
305   /**
306    * Load snapshot info and table descriptor for the specified snapshot
307    * @param snapshotName name of the snapshot to load
308    * @return false if snapshot is not found
309    */
310   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
311     snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
312     if (!fs.exists(snapshotDir)) {
313       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
314       return false;
315     }
316 
317     snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
318     snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
319     return true;
320   }
321 
322   /**
323    * Dump the {@link SnapshotDescription}
324    */
325   private void printInfo() {
326     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
327     System.out.println("Snapshot Info");
328     System.out.println("----------------------------------------");
329     System.out.println("   Name: " + snapshotDesc.getName());
330     System.out.println("   Type: " + snapshotDesc.getType());
331     System.out.println("  Table: " + snapshotTableDesc.getTableName().getNameAsString());
332     System.out.println(" Format: " + snapshotDesc.getVersion());
333     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
334     System.out.println();
335   }
336 
337   /**
338    * Dump the {@link HTableDescriptor}
339    */
340   private void printSchema() {
341     System.out.println("Table Descriptor");
342     System.out.println("----------------------------------------");
343     System.out.println(snapshotTableDesc.toString());
344     System.out.println();
345   }
346 
347   /**
348    * Collect the hfiles and logs statistics of the snapshot and
349    * dump the file list if requested and the collected information.
350    */
351   private void printFiles(final boolean showFiles) throws IOException {
352     if (showFiles) {
353       System.out.println("Snapshot Files");
354       System.out.println("----------------------------------------");
355     }
356 
357     // Collect information about hfiles and logs in the snapshot
358     final String table = snapshotTableDesc.getTableName().getNameAsString();
359     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
360     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
361       new SnapshotReferenceUtil.FileVisitor() {
362         public void storeFile (final String region, final String family, final String hfile)
363             throws IOException {
364           SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
365 
366           if (showFiles) {
367             System.out.printf("%8s %s/%s/%s/%s %s%n",
368               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
369               table, region, family, hfile,
370               (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
371           }
372         }
373 
374         public void recoveredEdits (final String region, final String logfile)
375             throws IOException {
376           SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
377 
378           if (showFiles) {
379             System.out.printf("%8s recovered.edits %s on region %s%n",
380               StringUtils.humanReadableInt(info.getSize()), logfile, region);
381           }
382         }
383 
384         public void logFile (final String server, final String logfile)
385             throws IOException {
386           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
387 
388           if (showFiles) {
389             System.out.printf("%8s log %s on server %s %s%n",
390               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
391               logfile, server,
392               (info.isMissing() ? "(NOT FOUND)" : ""));
393           }
394         }
395     });
396 
397     // Dump the stats
398     System.out.println();
399     if (stats.isSnapshotCorrupted()) {
400       System.out.println("**************************************************************");
401       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
402         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
403       System.out.println("**************************************************************");
404     }
405 
406     System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
407       stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
408       StringUtils.humanReadableInt(stats.getStoreFilesSize()),
409       stats.getSharedStoreFilePercentage(),
410       StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
411     );
412     System.out.printf("%d Logs, total size %s%n",
413       stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
414     System.out.println();
415   }
416 
417   private void printUsageAndExit() {
418     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
419     System.err.println(" where [options] are:");
420     System.err.println("  -h|-help                Show this help and exit.");
421     System.err.println("  -snapshot NAME          Snapshot to examine.");
422     System.err.println("  -files                  Files and logs list.");
423     System.err.println("  -stats                  Files and logs stats.");
424     System.err.println("  -schema                 Describe the snapshotted table.");
425     System.err.println();
426     System.err.println("Examples:");
427     System.err.println("  hbase " + getClass() + " \\");
428     System.err.println("    -snapshot MySnapshot -files");
429     System.exit(1);
430   }
431 
432   /**
433    * Returns the snapshot stats
434    * @param conf the {@link Configuration} to use
435    * @param snapshot {@link SnapshotDescription} to get stats from
436    * @return the snapshot stats
437    */
438   public static SnapshotStats getSnapshotStats(final Configuration conf,
439       final SnapshotDescription snapshot) throws IOException {
440     Path rootDir = FSUtils.getRootDir(conf);
441     FileSystem fs = FileSystem.get(conf);
442     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
443     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
444     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
445       new SnapshotReferenceUtil.FileVisitor() {
446         public void storeFile (final String region, final String family, final String hfile)
447             throws IOException {
448           stats.addStoreFile(region, family, hfile);
449         }
450 
451         public void recoveredEdits (final String region, final String logfile) throws IOException {
452           stats.addRecoveredEdits(region, logfile);
453         }
454 
455         public void logFile (final String server, final String logfile) throws IOException {
456           stats.addLogFile(server, logfile);
457         }
458     });
459     return stats;
460   }
461 
462   /**
463    * The guts of the {@link #main} method.
464    * Call this method to avoid the {@link #main(String[])} System.exit.
465    * @param args
466    * @return errCode
467    * @throws Exception
468    */
469   static int innerMain(final String [] args) throws Exception {
470     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
471   }
472 
473   public static void main(String[] args) throws Exception {
474      System.exit(innerMain(args));
475   }
476 }