View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.net.URI;
24  import java.text.SimpleDateFormat;
25  import java.util.ArrayList;
26  import java.util.Date;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.classification.InterfaceAudience;
36  import org.apache.hadoop.classification.InterfaceStability;
37  import org.apache.hadoop.conf.Configured;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40  import org.apache.hadoop.util.StringUtils;
41  import org.apache.hadoop.util.Tool;
42  import org.apache.hadoop.util.ToolRunner;
43  
44  import org.apache.hadoop.conf.Configuration;
45  import org.apache.hadoop.hbase.HBaseConfiguration;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.io.HFileLink;
48  import org.apache.hadoop.hbase.io.HLogLink;
49  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50  import org.apache.hadoop.hbase.util.FSUtils;
51  import org.apache.hadoop.hbase.util.FSTableDescriptors;
52  
53  /**
54   * Tool for dumping snapshot information.
55   * <ol>
56   * <li> Table Descriptor
57   * <li> Snapshot creation time, type, format version, ...
58   * <li> List of hfiles and hlogs
59   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
60   * </ol>
61   */
62  @InterfaceAudience.Public
63  @InterfaceStability.Evolving
64  public final class SnapshotInfo extends Configured implements Tool {
65    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
66  
67    /**
68     * Statistics about the snapshot
69     * <ol>
70     * <li> How many store files and logs are in the archive
71     * <li> How many store files and logs are shared with the table
72     * <li> Total store files and logs size and shared amount
73     * </ol>
74     */
75    public static class SnapshotStats {
76      /** Information about the file referenced by the snapshot */
77      static class FileInfo {
78        private final boolean inArchive;
79        private final long size;
80  
81        FileInfo(final boolean inArchive, final long size) {
82          this.inArchive = inArchive;
83          this.size = size;
84        }
85  
86        /** @return true if the file is in the archive */
87        public boolean inArchive() {
88          return this.inArchive;
89        }
90  
91        /** @return true if the file is missing */
92        public boolean isMissing() {
93          return this.size < 0;
94        }
95  
96        /** @return the file size */
97        public long getSize() {
98          return this.size;
99        }
100     }
101 
102     private int hfileArchiveCount = 0;
103     private int hfilesMissing = 0;
104     private int hfilesCount = 0;
105     private int logsMissing = 0;
106     private int logsCount = 0;
107     private long hfileArchiveSize = 0;
108     private long hfileSize = 0;
109     private long logSize = 0;
110 
111     private final SnapshotDescription snapshot;
112     private final TableName snapshotTable;
113     private final Configuration conf;
114     private final FileSystem fs;
115 
116     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
117     {
118       this.snapshot = snapshot;
119       this.snapshotTable = TableName.valueOf(snapshot.getTable());
120       this.conf = conf;
121       this.fs = fs;
122     }
123 
124     /** @return the snapshot descriptor */
125     public SnapshotDescription getSnapshotDescription() {
126       return this.snapshot;
127     }
128 
129     /** @return true if the snapshot is corrupted */
130     public boolean isSnapshotCorrupted() {
131       return hfilesMissing > 0 || logsMissing > 0;
132     }
133 
134     /** @return the number of available store files */
135     public int getStoreFilesCount() {
136       return hfilesCount + hfileArchiveCount;
137     }
138 
139     /** @return the number of available store files in the archive */
140     public int getArchivedStoreFilesCount() {
141       return hfileArchiveCount;
142     }
143 
144     /** @return the number of available log files */
145     public int getLogsCount() {
146       return logsCount;
147     }
148 
149     /** @return the number of missing store files */
150     public int getMissingStoreFilesCount() {
151       return hfilesMissing;
152     }
153 
154     /** @return the number of missing log files */
155     public int getMissingLogsCount() {
156       return logsMissing;
157     }
158 
159     /** @return the total size of the store files referenced by the snapshot */
160     public long getStoreFilesSize() {
161       return hfileSize + hfileArchiveSize;
162     }
163 
164     /** @return the total size of the store files shared */
165     public long getSharedStoreFilesSize() {
166       return hfileSize;
167     }
168 
169     /** @return the total size of the store files in the archive */
170     public long getArchivedStoreFileSize() {
171       return hfileArchiveSize;
172     }
173 
174     /** @return the percentage of the shared store files */
175     public float getSharedStoreFilePercentage() {
176       return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
177     }
178 
179     /** @return the total log size */
180     public long getLogsSize() {
181       return logSize;
182     }
183 
184     /**
185      * Add the specified store file to the stats
186      * @param region region encoded Name
187      * @param family family name
188      * @param hfile store file name
189      * @return the store file information
190      */
191     FileInfo addStoreFile(final String region, final String family, final String hfile)
192           throws IOException {
193       TableName table = snapshotTable;
194       HFileLink link = HFileLink.create(conf, table, region, family, hfile);
195       boolean inArchive = false;
196       long size = -1;
197       try {
198         if ((inArchive = fs.exists(link.getArchivePath()))) {
199           size = fs.getFileStatus(link.getArchivePath()).getLen();
200           hfileArchiveSize += size;
201           hfileArchiveCount++;
202         } else {
203           size = link.getFileStatus(fs).getLen();
204           hfileSize += size;
205           hfilesCount++;
206         }
207       } catch (FileNotFoundException e) {
208         hfilesMissing++;
209       }
210       return new FileInfo(inArchive, size);
211     }
212 
213     /**
214      * Add the specified recovered.edits file to the stats
215      * @param region region encoded name
216      * @param logfile log file name
217      * @return the recovered.edits information
218      */
219     FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
220       Path rootDir = FSUtils.getRootDir(conf);
221       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
222       Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
223       long size = fs.getFileStatus(path).getLen();
224       logSize += size;
225       logsCount++;
226       return new FileInfo(true, size);
227     }
228 
229     /**
230      * Add the specified log file to the stats
231      * @param server server name
232      * @param logfile log file name
233      * @return the log information
234      */
235     FileInfo addLogFile(final String server, final String logfile) throws IOException {
236       HLogLink logLink = new HLogLink(conf, server, logfile);
237       long size = -1;
238       try {
239         size = logLink.getFileStatus(fs).getLen();
240         logSize += size;
241         logsCount++;
242       } catch (FileNotFoundException e) {
243         logsMissing++;
244       }
245       return new FileInfo(false, size);
246     }
247   }
248 
249   private FileSystem fs;
250   private Path rootDir;
251 
252   private HTableDescriptor snapshotTableDesc;
253   private SnapshotDescription snapshotDesc;
254   private Path snapshotDir;
255 
256   @Override
257   public int run(String[] args) throws IOException, InterruptedException {
258     final Configuration conf = getConf();
259     boolean listSnapshots = false;
260     String snapshotName = null;
261     boolean showSchema = false;
262     boolean showFiles = false;
263     boolean showStats = false;
264 
265     // Process command line args
266     for (int i = 0; i < args.length; i++) {
267       String cmd = args[i];
268       try {
269         if (cmd.equals("-snapshot")) {
270           snapshotName = args[++i];
271         } else if (cmd.equals("-files")) {
272           showFiles = true;
273           showStats = true;
274         } else if (cmd.equals("-stats")) {
275           showStats = true;
276         } else if (cmd.equals("-schema")) {
277           showSchema = true;
278         } else if (cmd.equals("-remote-dir")) {
279           Path sourceDir = new Path(args[++i]);
280           URI defaultFs = sourceDir.getFileSystem(conf).getUri();
281           FSUtils.setFsDefault(conf, new Path(defaultFs));
282           FSUtils.setRootDir(conf, sourceDir);
283         } else if (cmd.equals("-list-snapshots")) {
284           listSnapshots = true;
285         } else if (cmd.equals("-h") || cmd.equals("--help")) {
286           printUsageAndExit();
287         } else {
288           System.err.println("UNEXPECTED: " + cmd);
289           printUsageAndExit();
290         }
291       } catch (Exception e) {
292         printUsageAndExit();
293       }
294     }
295 
296     // List Available Snapshots
297     if (listSnapshots) {
298       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
299       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
300       for (SnapshotDescription desc: getSnapshotList(conf)) {
301         System.out.printf("%-20s | %20s | %s%n",
302                           desc.getName(),
303                           df.format(new Date(desc.getCreationTime())),
304                           desc.getTable());
305       }
306       return 0;
307     }
308 
309     if (snapshotName == null) {
310       System.err.println("Missing snapshot name!");
311       printUsageAndExit();
312       return 1;
313     }
314 
315     rootDir = FSUtils.getRootDir(conf);
316     fs = FileSystem.get(rootDir.toUri(), conf);
317     LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
318 
319     // Load snapshot information
320     if (!loadSnapshotInfo(snapshotName)) {
321       System.err.println("Snapshot '" + snapshotName + "' not found!");
322       return 1;
323     }
324 
325     printInfo();
326     if (showSchema) printSchema();
327     printFiles(showFiles, showStats);
328 
329     return 0;
330   }
331 
332   /**
333    * Load snapshot info and table descriptor for the specified snapshot
334    * @param snapshotName name of the snapshot to load
335    * @return false if snapshot is not found
336    */
337   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
338     snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
339     if (!fs.exists(snapshotDir)) {
340       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
341       return false;
342     }
343 
344     snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
345     snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
346     return true;
347   }
348 
349   /**
350    * Dump the {@link SnapshotDescription}
351    */
352   private void printInfo() {
353     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
354     System.out.println("Snapshot Info");
355     System.out.println("----------------------------------------");
356     System.out.println("   Name: " + snapshotDesc.getName());
357     System.out.println("   Type: " + snapshotDesc.getType());
358     System.out.println("  Table: " + snapshotTableDesc.getTableName().getNameAsString());
359     System.out.println(" Format: " + snapshotDesc.getVersion());
360     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
361     System.out.println();
362   }
363 
364   /**
365    * Dump the {@link HTableDescriptor}
366    */
367   private void printSchema() {
368     System.out.println("Table Descriptor");
369     System.out.println("----------------------------------------");
370     System.out.println(snapshotTableDesc.toString());
371     System.out.println();
372   }
373 
374   /**
375    * Collect the hfiles and logs statistics of the snapshot and
376    * dump the file list if requested and the collected information.
377    */
378   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
379     if (showFiles) {
380       System.out.println("Snapshot Files");
381       System.out.println("----------------------------------------");
382     }
383 
384     // Collect information about hfiles and logs in the snapshot
385     final String table = snapshotTableDesc.getTableName().getNameAsString();
386     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
387     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
388       new SnapshotReferenceUtil.FileVisitor() {
389         public void storeFile (final String region, final String family, final String hfile)
390             throws IOException {
391           SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
392 
393           if (showFiles) {
394             System.out.printf("%8s %s/%s/%s/%s %s%n",
395               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
396               table, region, family, hfile,
397               (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
398           }
399         }
400 
401         public void recoveredEdits (final String region, final String logfile)
402             throws IOException {
403           SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
404 
405           if (showFiles) {
406             System.out.printf("%8s recovered.edits %s on region %s%n",
407               StringUtils.humanReadableInt(info.getSize()), logfile, region);
408           }
409         }
410 
411         public void logFile (final String server, final String logfile)
412             throws IOException {
413           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
414 
415           if (showFiles) {
416             System.out.printf("%8s log %s on server %s %s%n",
417               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
418               logfile, server,
419               (info.isMissing() ? "(NOT FOUND)" : ""));
420           }
421         }
422     });
423 
424     // Dump the stats
425     System.out.println();
426     if (stats.isSnapshotCorrupted()) {
427       System.out.println("**************************************************************");
428       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
429         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
430       System.out.println("**************************************************************");
431     }
432 
433     if (showStats) {
434       System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
435         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
436         StringUtils.humanReadableInt(stats.getStoreFilesSize()),
437         stats.getSharedStoreFilePercentage(),
438         StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
439       );
440       System.out.printf("%d Logs, total size %s%n",
441         stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
442       System.out.println();
443     }
444   }
445 
446   private void printUsageAndExit() {
447     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
448     System.err.println(" where [options] are:");
449     System.err.println("  -h|-help                Show this help and exit.");
450     System.err.println("  -remote-dir             Root directory that contains the snapshots.");
451     System.err.println("  -list-snapshots         List all the available snapshots and exit.");
452     System.err.println("  -snapshot NAME          Snapshot to examine.");
453     System.err.println("  -files                  Files and logs list.");
454     System.err.println("  -stats                  Files and logs stats.");
455     System.err.println("  -schema                 Describe the snapshotted table.");
456     System.err.println();
457     System.err.println("Examples:");
458     System.err.println("  hbase " + getClass() + " \\");
459     System.err.println("    -snapshot MySnapshot -files");
460     System.exit(1);
461   }
462 
463   /**
464    * Returns the snapshot stats
465    * @param conf the {@link Configuration} to use
466    * @param snapshot {@link SnapshotDescription} to get stats from
467    * @return the snapshot stats
468    */
469   public static SnapshotStats getSnapshotStats(final Configuration conf,
470       final SnapshotDescription snapshot) throws IOException {
471     Path rootDir = FSUtils.getRootDir(conf);
472     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
473     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
474     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
475     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
476       new SnapshotReferenceUtil.FileVisitor() {
477         public void storeFile (final String region, final String family, final String hfile)
478             throws IOException {
479           stats.addStoreFile(region, family, hfile);
480         }
481 
482         public void recoveredEdits (final String region, final String logfile) throws IOException {
483           stats.addRecoveredEdits(region, logfile);
484         }
485 
486         public void logFile (final String server, final String logfile) throws IOException {
487           stats.addLogFile(server, logfile);
488         }
489     });
490     return stats;
491   }
492 
493   /**
494    * Returns the list of available snapshots in the specified location
495    * @param conf the {@link Configuration} to use
496    * @return the list of snapshots
497    */
498   public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
499       throws IOException {
500     Path rootDir = FSUtils.getRootDir(conf);
501     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
502     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
503     FileStatus[] snapshots = fs.listStatus(snapshotDir,
504       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
505     List<SnapshotDescription> snapshotLists =
506       new ArrayList<SnapshotDescription>(snapshots.length);
507     for (FileStatus snapshotDirStat: snapshots) {
508       snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
509     }
510     return snapshotLists;
511   }
512 
513   /**
514    * The guts of the {@link #main} method.
515    * Call this method to avoid the {@link #main(String[])} System.exit.
516    * @param args
517    * @return errCode
518    * @throws Exception
519    */
520   static int innerMain(final String [] args) throws Exception {
521     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
522   }
523 
524   public static void main(String[] args) throws Exception {
525      System.exit(innerMain(args));
526   }
527 }