View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.net.URI;
24  import java.text.SimpleDateFormat;
25  import java.util.concurrent.atomic.AtomicInteger;
26  import java.util.concurrent.atomic.AtomicLong;
27  import java.util.ArrayList;
28  import java.util.Date;
29  import java.util.List;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.classification.InterfaceAudience;
38  import org.apache.hadoop.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configured;
40  import org.apache.hadoop.util.StringUtils;
41  import org.apache.hadoop.util.Tool;
42  import org.apache.hadoop.util.ToolRunner;
43  
44  import org.apache.hadoop.conf.Configuration;
45  import org.apache.hadoop.hbase.HBaseConfiguration;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.io.HFileLink;
48  import org.apache.hadoop.hbase.io.HLogLink;
49  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
51  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.FSUtils;
54  import org.apache.hadoop.hbase.util.FSTableDescriptors;
55  
56  /**
57   * Tool for dumping snapshot information.
58   * <ol>
59   * <li> Table Descriptor
60   * <li> Snapshot creation time, type, format version, ...
61   * <li> List of hfiles and hlogs
62   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
63   * </ol>
64   */
65  @InterfaceAudience.Public
66  @InterfaceStability.Evolving
67  public final class SnapshotInfo extends Configured implements Tool {
68    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
69  
70    /**
71     * Statistics about the snapshot
72     * <ol>
73     * <li> How many store files and logs are in the archive
74     * <li> How many store files and logs are shared with the table
75     * <li> Total store files and logs size and shared amount
76     * </ol>
77     */
78    public static class SnapshotStats {
79      /** Information about the file referenced by the snapshot */
80      static class FileInfo {
81        private final boolean inArchive;
82        private final long size;
83  
84        FileInfo(final boolean inArchive, final long size) {
85          this.inArchive = inArchive;
86          this.size = size;
87        }
88  
89        /** @return true if the file is in the archive */
90        public boolean inArchive() {
91          return this.inArchive;
92        }
93  
94        /** @return true if the file is missing */
95        public boolean isMissing() {
96          return this.size < 0;
97        }
98  
99        /** @return the file size */
100       public long getSize() {
101         return this.size;
102       }
103     }
104 
105     private int hfileArchiveCount = 0;
106     private int hfilesMissing = 0;
107     private int hfilesCount = 0;
108     private int logsMissing = 0;
109     private int logsCount = 0;
110     private long hfileArchiveSize = 0;
111     private long hfileSize = 0;
112     private long logSize = 0;
113 
114     private final SnapshotDescription snapshot;
115     private final Configuration conf;
116     private final FileSystem fs;
117 
118     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
119     {
120       this.snapshot = snapshot;
121       this.conf = conf;
122       this.fs = fs;
123     }
124 
125     /** @return the snapshot descriptor */
126     public SnapshotDescription getSnapshotDescription() {
127       return this.snapshot;
128     }
129 
130     /** @return true if the snapshot is corrupted */
131     public boolean isSnapshotCorrupted() {
132       return hfilesMissing > 0 || logsMissing > 0;
133     }
134 
135     /** @return the number of available store files */
136     public int getStoreFilesCount() {
137       return hfilesCount + hfileArchiveCount;
138     }
139 
140     /** @return the number of available store files in the archive */
141     public int getArchivedStoreFilesCount() {
142       return hfileArchiveCount;
143     }
144 
145     /** @return the number of available log files */
146     public int getLogsCount() {
147       return logsCount;
148     }
149 
150     /** @return the number of missing store files */
151     public int getMissingStoreFilesCount() {
152       return hfilesMissing;
153     }
154 
155     /** @return the number of missing log files */
156     public int getMissingLogsCount() {
157       return logsMissing;
158     }
159 
160     /** @return the total size of the store files referenced by the snapshot */
161     public long getStoreFilesSize() {
162       return hfileSize + hfileArchiveSize;
163     }
164 
165     /** @return the total size of the store files shared */
166     public long getSharedStoreFilesSize() {
167       return hfileSize;
168     }
169 
170     /** @return the total size of the store files in the archive */
171     public long getArchivedStoreFileSize() {
172       return hfileArchiveSize;
173     }
174 
175     /** @return the percentage of the shared store files */
176     public float getSharedStoreFilePercentage() {
177       return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
178     }
179 
180     /** @return the total log size */
181     public long getLogsSize() {
182       return logSize;
183     }
184 
185     /**
186      * Add the specified store file to the stats
187      * @param region region encoded Name
188      * @param family family name
189      * @param hfile store file name
190      * @return the store file information
191      */
192     FileInfo addStoreFile(final String region, final String family, final String hfile)
193           throws IOException {
194       String table = this.snapshot.getTable();
195       HFileLink link = HFileLink.create(conf, table, region, family, hfile);
196       boolean inArchive = false;
197       long size = -1;
198       try {
199         if ((inArchive = fs.exists(link.getArchivePath()))) {
200           size = fs.getFileStatus(link.getArchivePath()).getLen();
201           hfileArchiveSize += size;
202           hfileArchiveCount++;
203         } else {
204           size = link.getFileStatus(fs).getLen();
205           hfileSize += size;
206           hfilesCount++;
207         }
208       } catch (FileNotFoundException e) {
209         hfilesMissing++;
210       }
211       return new FileInfo(inArchive, size);
212     }
213 
214     /**
215      * Add the specified recovered.edits file to the stats
216      * @param region region encoded name
217      * @param logfile log file name
218      * @return the recovered.edits information
219      */
220     FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
221       Path rootDir = FSUtils.getRootDir(conf);
222       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
223       Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
224       long size = fs.getFileStatus(path).getLen();
225       logSize += size;
226       logsCount++;
227       return new FileInfo(true, size);
228     }
229 
230     /**
231      * Add the specified log file to the stats
232      * @param server server name
233      * @param logfile log file name
234      * @return the log information
235      */
236     FileInfo addLogFile(final String server, final String logfile) throws IOException {
237       HLogLink logLink = new HLogLink(conf, server, logfile);
238       long size = -1;
239       try {
240         size = logLink.getFileStatus(fs).getLen();
241         logSize += size;
242         logsCount++;
243       } catch (FileNotFoundException e) {
244         logsMissing++;
245       }
246       return new FileInfo(false, size);
247     }
248   }
249 
250   private FileSystem fs;
251   private Path rootDir;
252 
253   private HTableDescriptor snapshotTableDesc;
254   private SnapshotDescription snapshotDesc;
255   private Path snapshotDir;
256 
257   @Override
258   public int run(String[] args) throws IOException, InterruptedException {
259     final Configuration conf = getConf();
260     boolean listSnapshots = false;
261     String snapshotName = null;
262     boolean showSchema = false;
263     boolean showFiles = false;
264     boolean showStats = false;
265 
266     // Process command line args
267     for (int i = 0; i < args.length; i++) {
268       String cmd = args[i];
269       try {
270         if (cmd.equals("-snapshot")) {
271           snapshotName = args[++i];
272         } else if (cmd.equals("-files")) {
273           showFiles = true;
274           showStats = true;
275         } else if (cmd.equals("-stats")) {
276           showStats = true;
277         } else if (cmd.equals("-schema")) {
278           showSchema = true;
279         } else if (cmd.equals("-remote-dir")) {
280           Path sourceDir = new Path(args[++i]);
281           URI defaultFs = sourceDir.getFileSystem(conf).getUri();
282           FSUtils.setFsDefault(conf, new Path(defaultFs));
283           FSUtils.setRootDir(conf, sourceDir);
284         } else if (cmd.equals("-list-snapshots")) {
285           listSnapshots = true;
286         } else if (cmd.equals("-h") || cmd.equals("--help")) {
287           printUsageAndExit();
288         } else {
289           System.err.println("UNEXPECTED: " + cmd);
290           printUsageAndExit();
291         }
292       } catch (Exception e) {
293         printUsageAndExit();
294       }
295     }
296 
297     // List Available Snapshots
298     if (listSnapshots) {
299       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
300       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
301       for (SnapshotDescription desc: getSnapshotList(conf)) {
302         System.out.printf("%-20s | %20s | %s%n",
303                           desc.getName(),
304                           df.format(new Date(desc.getCreationTime())),
305                           desc.getTable());
306       }
307       return 0;
308     }
309 
310     if (snapshotName == null) {
311       System.err.println("Missing snapshot name!");
312       printUsageAndExit();
313       return 1;
314     }
315 
316     rootDir = FSUtils.getRootDir(conf);
317     fs = FileSystem.get(rootDir.toUri(), conf);
318     LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
319 
320     // Load snapshot information
321     if (!loadSnapshotInfo(snapshotName)) {
322       System.err.println("Snapshot '" + snapshotName + "' not found!");
323       return 1;
324     }
325 
326     printInfo();
327     if (showSchema) printSchema();
328     printFiles(showFiles, showStats);
329 
330     return 0;
331   }
332 
333   /**
334    * Load snapshot info and table descriptor for the specified snapshot
335    * @param snapshotName name of the snapshot to load
336    * @return false if snapshot is not found
337    */
338   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
339     snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
340     if (!fs.exists(snapshotDir)) {
341       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
342       return false;
343     }
344 
345     snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
346     snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
347     return true;
348   }
349 
350   /**
351    * Dump the {@link SnapshotDescription}
352    */
353   private void printInfo() {
354     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
355     System.out.println("Snapshot Info");
356     System.out.println("----------------------------------------");
357     System.out.println("   Name: " + snapshotDesc.getName());
358     System.out.println("   Type: " + snapshotDesc.getType());
359     System.out.println("  Table: " + snapshotDesc.getTable());
360     System.out.println(" Format: " + snapshotDesc.getVersion());
361     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
362     System.out.println();
363   }
364 
365   /**
366    * Dump the {@link HTableDescriptor}
367    */
368   private void printSchema() {
369     System.out.println("Table Descriptor");
370     System.out.println("----------------------------------------");
371     System.out.println(snapshotTableDesc.toString());
372     System.out.println();
373   }
374 
375   /**
376    * Collect the hfiles and logs statistics of the snapshot and
377    * dump the file list if requested and the collected information.
378    */
379   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
380     if (showFiles) {
381       System.out.println("Snapshot Files");
382       System.out.println("----------------------------------------");
383     }
384 
385     // Collect information about hfiles and logs in the snapshot
386     final String table = this.snapshotDesc.getTable();
387     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
388     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
389       new SnapshotReferenceUtil.FileVisitor() {
390         public void storeFile (final String region, final String family, final String hfile)
391             throws IOException {
392           SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
393 
394           if (showFiles) {
395             System.out.printf("%8s %s/%s/%s/%s %s%n",
396               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
397               table, region, family, hfile,
398               (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : ""));
399           }
400         }
401 
402         public void recoveredEdits (final String region, final String logfile)
403             throws IOException {
404           SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
405 
406           if (showFiles) {
407             System.out.printf("%8s recovered.edits %s on region %s%n",
408               StringUtils.humanReadableInt(info.getSize()), logfile, region);
409           }
410         }
411 
412         public void logFile (final String server, final String logfile)
413             throws IOException {
414           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
415 
416           if (showFiles) {
417             System.out.printf("%8s log %s on server %s %s%n",
418               (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())),
419               logfile, server,
420               (info.isMissing() ? "(NOT FOUND)" : ""));
421           }
422         }
423     });
424 
425     // Dump the stats
426     System.out.println();
427     if (stats.isSnapshotCorrupted()) {
428       System.out.println("**************************************************************");
429       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
430         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
431       System.out.println("**************************************************************");
432     }
433 
434     if (showStats) {
435       System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
436         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
437         StringUtils.humanReadableInt(stats.getStoreFilesSize()),
438         stats.getSharedStoreFilePercentage(),
439         StringUtils.humanReadableInt(stats.getSharedStoreFilesSize())
440       );
441       System.out.printf("%d Logs, total size %s%n",
442         stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize()));
443       System.out.println();
444     }
445   }
446 
447   private void printUsageAndExit() {
448     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
449     System.err.println(" where [options] are:");
450     System.err.println("  -h|-help                Show this help and exit.");
451     System.err.println("  -remote-dir             Root directory that contains the snapshots.");
452     System.err.println("  -list-snapshots         List all the available snapshots and exit.");
453     System.err.println("  -snapshot NAME          Snapshot to examine.");
454     System.err.println("  -files                  Files and logs list.");
455     System.err.println("  -stats                  Files and logs stats.");
456     System.err.println("  -schema                 Describe the snapshotted table.");
457     System.err.println();
458     System.err.println("Examples:");
459     System.err.println("  hbase " + getClass() + " \\");
460     System.err.println("    -snapshot MySnapshot -files");
461     System.exit(1);
462   }
463 
464   /**
465    * Returns the snapshot stats
466    * @param conf the {@link Configuration} to use
467    * @param snapshot {@link SnapshotDescription} to get stats from
468    * @return the snapshot stats
469    */
470   public static SnapshotStats getSnapshotStats(final Configuration conf,
471       final SnapshotDescription snapshot) throws IOException {
472     Path rootDir = FSUtils.getRootDir(conf);
473     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
474     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
475     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
476     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
477       new SnapshotReferenceUtil.FileVisitor() {
478         public void storeFile (final String region, final String family, final String hfile)
479             throws IOException {
480           stats.addStoreFile(region, family, hfile);
481         }
482 
483         public void recoveredEdits (final String region, final String logfile) throws IOException {
484           stats.addRecoveredEdits(region, logfile);
485         }
486 
487         public void logFile (final String server, final String logfile) throws IOException {
488           stats.addLogFile(server, logfile);
489         }
490     });
491     return stats;
492   }
493 
494   /**
495    * Returns the list of available snapshots in the specified location
496    * @param conf the {@link Configuration} to use
497    * @return the list of snapshots
498    */
499   public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
500       throws IOException {
501     Path rootDir = FSUtils.getRootDir(conf);
502     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
503     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
504     FileStatus[] snapshots = fs.listStatus(snapshotDir,
505       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
506     List<SnapshotDescription> snapshotLists =
507       new ArrayList<SnapshotDescription>(snapshots.length);
508     for (FileStatus snapshotDirStat: snapshots) {
509       snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
510     }
511     return snapshotLists;
512   }
513 
514   /**
515    * The guts of the {@link #main} method.
516    * Call this method to avoid the {@link #main(String[])} System.exit.
517    * @param args
518    * @return errCode
519    * @throws Exception
520    */
521   static int innerMain(final String [] args) throws Exception {
522     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
523   }
524 
525   public static void main(String[] args) throws Exception {
526      System.exit(innerMain(args));
527   }
528 }