View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.text.SimpleDateFormat;
24  import java.util.concurrent.atomic.AtomicInteger;
25  import java.util.concurrent.atomic.AtomicLong;
26  import java.util.Date;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.classification.InterfaceAudience;
34  import org.apache.hadoop.classification.InterfaceStability;
35  import org.apache.hadoop.conf.Configured;
36  import org.apache.hadoop.util.StringUtils;
37  import org.apache.hadoop.util.Tool;
38  import org.apache.hadoop.util.ToolRunner;
39  
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.io.HFileLink;
44  import org.apache.hadoop.hbase.io.HLogLink;
45  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
47  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.FSUtils;
50  import org.apache.hadoop.hbase.util.FSTableDescriptors;
51  
52  /**
53   * Tool for dumping snapshot information.
54   * <ol>
55   * <li> Table Descriptor
56   * <li> Snapshot creation time, type, format version, ...
57   * <li> List of hfiles and hlogs
58   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
59   * </ol>
60   */
61  @InterfaceAudience.Public
62  @InterfaceStability.Evolving
63  public final class SnapshotInfo extends Configured implements Tool {
64    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
65  
66    /**
67     * Statistics about the snapshot
68     * <ol>
69     * <li> How many store files and logs are in the archive
70     * <li> How many store files and logs are shared with the table
71     * <li> Total store files and logs size and shared amount
72     * </ol>
73     */
74    public static class SnapshotStats {
75      /** Information about the file referenced by the snapshot */
76      static class FileInfo {
77        private final boolean inArchive;
78        private final long size;
79  
80        FileInfo(final boolean inArchive, final long size) {
81          this.inArchive = inArchive;
82          this.size = size;
83        }
84  
85        /** @return true if the file is in the archive */
86        public boolean inArchive() {
87          return this.inArchive;
88        }
89  
90        /** @return true if the file is missing */
91        public boolean isMissing() {
92          return this.size < 0;
93        }
94  
95        /** @return the file size */
96        public long getSize() {
97          return this.size;
98        }
99      }
100 
101     private int hfileArchiveCount = 0;
102     private int hfilesMissing = 0;
103     private int hfilesCount = 0;
104     private int logsMissing = 0;
105     private int logsCount = 0;
106     private long hfileArchiveSize = 0;
107     private long hfileSize = 0;
108     private long logSize = 0;
109 
110     private final SnapshotDescription snapshot;
111     private final Configuration conf;
112     private final FileSystem fs;
113 
114     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
115     {
116       this.snapshot = snapshot;
117       this.conf = conf;
118       this.fs = fs;
119     }
120 
121     /** @return the snapshot descriptor */
122     public SnapshotDescription getSnapshotDescription() {
123       return this.snapshot;
124     }
125 
126     /** @return true if the snapshot is corrupted */
127     public boolean isSnapshotCorrupted() {
128       return hfilesMissing > 0 || logsMissing > 0;
129     }
130 
131     /** @return the number of available store files */
132     public int getStoreFilesCount() {
133       return hfilesCount + hfileArchiveCount;
134     }
135 
136     /** @return the number of available store files in the archive */
137     public int getArchivedStoreFilesCount() {
138       return hfileArchiveCount;
139     }
140 
141     /** @return the number of available log files */
142     public int getLogsCount() {
143       return logsCount;
144     }
145 
146     /** @return the number of missing store files */
147     public int getMissingStoreFilesCount() {
148       return hfilesMissing;
149     }
150 
151     /** @return the number of missing log files */
152     public int getMissingLogsCount() {
153       return logsMissing;
154     }
155 
156     /** @return the total size of the store files referenced by the snapshot */
157     public long getStoreFilesSize() {
158       return hfileSize + hfileArchiveSize;
159     }
160 
161     /** @return the total size of the store files shared */
162     public long getSharedStoreFilesSize() {
163       return hfileSize;
164     }
165 
166     /** @return the total size of the store files in the archive */
167     public long getArchivedStoreFileSize() {
168       return hfileArchiveSize;
169     }
170 
171     /** @return the percentage of the shared store files */
172     public float getSharedStoreFilePercentage() {
173       return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
174     }
175 
176     /** @return the total log size */
177     public long getLogsSize() {
178       return logSize;
179     }
180 
181     /**
182      * Add the specified store file to the stats
183      * @param region region encoded Name
184      * @param family family name
185      * @param hfile store file name
186      * @return the store file information
187      */
188     FileInfo addStoreFile(final String region, final String family, final String hfile)
189           throws IOException {
190       String table = this.snapshot.getTable();
191       HFileLink link = HFileLink.create(conf, table, region, family, hfile);
192       boolean inArchive = false;
193       long size = -1;
194       try {
195         if ((inArchive = fs.exists(link.getArchivePath()))) {
196           size = fs.getFileStatus(link.getArchivePath()).getLen();
197           hfileArchiveSize += size;
198           hfileArchiveCount++;
199         } else {
200           size = link.getFileStatus(fs).getLen();
201           hfileSize += size;
202           hfilesCount++;
203         }
204       } catch (FileNotFoundException e) {
205         hfilesMissing++;
206       }
207       return new FileInfo(inArchive, size);
208     }
209 
210     /**
211      * Add the specified recovered.edits file to the stats
212      * @param region region encoded name
213      * @param logfile log file name
214      * @return the recovered.edits information
215      */
216     FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
217       Path rootDir = FSUtils.getRootDir(conf);
218       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
219       Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
220       long size = fs.getFileStatus(path).getLen();
221       logSize += size;
222       logsCount++;
223       return new FileInfo(true, size);
224     }
225 
226     /**
227      * Add the specified log file to the stats
228      * @param server server name
229      * @param logfile log file name
230      * @return the log information
231      */
232     FileInfo addLogFile(final String server, final String logfile) throws IOException {
233       HLogLink logLink = new HLogLink(conf, server, logfile);
234       long size = -1;
235       try {
236         size = logLink.getFileStatus(fs).getLen();
237         logSize += size;
238         logsCount++;
239       } catch (FileNotFoundException e) {
240         logsMissing++;
241       }
242       return new FileInfo(false, size);
243     }
244   }
245 
246   private FileSystem fs;
247   private Path rootDir;
248 
249   private HTableDescriptor snapshotTableDesc;
250   private SnapshotDescription snapshotDesc;
251   private Path snapshotDir;
252 
253   @Override
254   public int run(String[] args) throws IOException, InterruptedException {
255     String snapshotName = null;
256     boolean showSchema = false;
257     boolean showFiles = false;
258     boolean showStats = false;
259 
260     // Process command line args
261     for (int i = 0; i < args.length; i++) {
262       String cmd = args[i];
263       try {
264         if (cmd.equals("-snapshot")) {
265           snapshotName = args[++i];
266         } else if (cmd.equals("-files")) {
267           showFiles = true;
268           showStats = true;
269         } else if (cmd.equals("-stats")) {
270           showStats = true;
271         } else if (cmd.equals("-schema")) {
272           showSchema = true;
273         } else if (cmd.equals("-h") || cmd.equals("--help")) {
274           printUsageAndExit();
275         } else {
276           System.err.println("UNEXPECTED: " + cmd);
277           printUsageAndExit();
278         }
279       } catch (Exception e) {
280         printUsageAndExit();
281       }
282     }
283 
284     if (snapshotName == null) {
285       System.err.println("Missing snapshot name!");
286       printUsageAndExit();
287       return 1;
288     }
289 
290     Configuration conf = getConf();
291     fs = FileSystem.get(conf);
292     rootDir = FSUtils.getRootDir(conf);
293 
294     // Load snapshot information
295     if (!loadSnapshotInfo(snapshotName)) {
296       System.err.println("Snapshot '" + snapshotName + "' not found!");
297       return 1;
298     }
299 
300     printInfo();
301     if (showSchema) printSchema();
302     printFiles(showFiles, showStats);
303 
304     return 0;
305   }
306 
307   /**
308    * Load snapshot info and table descriptor for the specified snapshot
309    * @param snapshotName name of the snapshot to load
310    * @return false if snapshot is not found
311    */
312   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
313     snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
314     if (!fs.exists(snapshotDir)) {
315       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
316       return false;
317     }
318 
319     snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
320     snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
321     return true;
322   }
323 
324   /**
325    * Dump the {@link SnapshotDescription}
326    */
327   private void printInfo() {
328     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
329     System.out.println("Snapshot Info");
330     System.out.println("----------------------------------------");
331     System.out.println("   Name: " + snapshotDesc.getName());
332     System.out.println("   Type: " + snapshotDesc.getType());
333     System.out.println("  Table: " + snapshotDesc.getTable());
334     System.out.println(" Format: " + snapshotDesc.getVersion());
335     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
336     System.out.println();
337   }
338 
339   /**
340    * Dump the {@link HTableDescriptor}
341    */
342   private void printSchema() {
343     System.out.println("Table Descriptor");
344     System.out.println("----------------------------------------");
345     System.out.println(snapshotTableDesc.toString());
346     System.out.println();
347   }
348 
349   /**
350    * Collect the hfiles and logs statistics of the snapshot and
351    * dump the file list if requested and the collected information.
352    */
353   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
354     if (showFiles) {
355       System.out.println("Snapshot Files");
356       System.out.println("----------------------------------------");
357     }
358 
359     // Collect information about hfiles and logs in the snapshot
360     final String table = this.snapshotDesc.getTable();
361     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
362     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
363       new SnapshotReferenceUtil.FileVisitor() {
364         public void storeFile (final String region, final String family, final String hfile)
365             throws IOException {
366           SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
367 
368           if (showFiles) {
369             System.out.printf("%8s %s/%s/%s/%s %s%n",
370               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
371               table, region, family, hfile,
372               (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
373           }
374         }
375 
376         public void recoveredEdits (final String region, final String logfile)
377             throws IOException {
378           SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
379 
380           if (showFiles) {
381             System.out.printf("%8s recovered.edits %s on region %s%n",
382               StringUtils.humanReadableInt(info.getSize()), logfile, region);
383           }
384         }
385 
386         public void logFile (final String server, final String logfile)
387             throws IOException {
388           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
389 
390           if (showFiles) {
391             System.out.printf("%8s log %s on server %s %s%n",
392               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
393               logfile, server,
394               (info.isMissing() ? "(NOT FOUND)" : ""));
395           }
396         }
397     });
398 
399     // Dump the stats
400     System.out.println();
401     if (stats.isSnapshotCorrupted()) {
402       System.out.println("**************************************************************");
403       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
404         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
405       System.out.println("**************************************************************");
406     }
407 
408     if (showStats) {
409       System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
410         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
411         StringUtils.humanReadableInt(stats.getStoreFilesSize()),
412         stats.getSharedStoreFilePercentage(),
413         StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
414       );
415       System.out.printf("%d Logs, total size %s%n",
416         stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
417       System.out.println();
418     }
419   }
420 
421   private void printUsageAndExit() {
422     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
423     System.err.println(" where [options] are:");
424     System.err.println("  -h|-help                Show this help and exit.");
425     System.err.println("  -snapshot NAME          Snapshot to examine.");
426     System.err.println("  -files                  Files and logs list.");
427     System.err.println("  -stats                  Files and logs stats.");
428     System.err.println("  -schema                 Describe the snapshotted table.");
429     System.err.println();
430     System.err.println("Examples:");
431     System.err.println("  hbase " + getClass() + " \\");
432     System.err.println("    -snapshot MySnapshot -files");
433     System.exit(1);
434   }
435 
436   /**
437    * Returns the snapshot stats
438    * @param conf the {@link Configuration} to use
439    * @param snapshot {@link SnapshotDescription} to get stats from
440    * @return the snapshot stats
441    */
442   public static SnapshotStats getSnapshotStats(final Configuration conf,
443       final SnapshotDescription snapshot) throws IOException {
444     Path rootDir = FSUtils.getRootDir(conf);
445     FileSystem fs = FileSystem.get(conf);
446     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
447     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
448     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
449       new SnapshotReferenceUtil.FileVisitor() {
450         public void storeFile (final String region, final String family, final String hfile)
451             throws IOException {
452           stats.addStoreFile(region, family, hfile);
453         }
454 
455         public void recoveredEdits (final String region, final String logfile) throws IOException {
456           stats.addRecoveredEdits(region, logfile);
457         }
458 
459         public void logFile (final String server, final String logfile) throws IOException {
460           stats.addLogFile(server, logfile);
461         }
462     });
463     return stats;
464   }
465 
466   /**
467    * The guts of the {@link #main} method.
468    * Call this method to avoid the {@link #main(String[])} System.exit.
469    * @param args
470    * @return errCode
471    * @throws Exception
472    */
473   static int innerMain(final String [] args) throws Exception {
474     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
475   }
476 
477   public static void main(String[] args) throws Exception {
478      System.exit(innerMain(args));
479   }
480 }