View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.util;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FSDataInputStream;
26  import org.apache.hadoop.fs.FSDataOutputStream;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.fs.PathFilter;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.RemoteExceptionHandler;
34  import org.apache.hadoop.hbase.master.HMaster;
35  import org.apache.hadoop.hbase.regionserver.HRegion;
36  import org.apache.hadoop.hdfs.DistributedFileSystem;
37  import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
38  import org.apache.hadoop.hdfs.protocol.FSConstants;
39  import org.apache.hadoop.io.SequenceFile;
40  
41  import java.io.DataInputStream;
42  import java.io.IOException;
43  import java.net.URI;
44  import java.net.URISyntaxException;
45  import java.util.HashMap;
46  import java.util.Map;
47  
48  /**
49   * Utility methods for interacting with the underlying file system.
50   */
51  public class FSUtils {
52    private static final Log LOG = LogFactory.getLog(FSUtils.class);
53  
54    /**
55     * Not instantiable
56     */
57    private FSUtils() {
58      super();
59    }
60  
61    /**
62     * Delete if exists.
63     * @param fs filesystem object
64     * @param dir directory to delete
65     * @return True if deleted <code>dir</code>
66     * @throws IOException e
67     */
68    public static boolean deleteDirectory(final FileSystem fs, final Path dir)
69    throws IOException {
70      return fs.exists(dir) && fs.delete(dir, true);
71    }
72  
73    /**
74     * Check if directory exists.  If it does not, create it.
75     * @param fs filesystem object
76     * @param dir path to check
77     * @return Path
78     * @throws IOException e
79     */
80    public Path checkdir(final FileSystem fs, final Path dir) throws IOException {
81      if (!fs.exists(dir)) {
82        fs.mkdirs(dir);
83      }
84      return dir;
85    }
86  
87    /**
88     * Create file.
89     * @param fs filesystem object
90     * @param p path to create
91     * @return Path
92     * @throws IOException e
93     */
94    public static Path create(final FileSystem fs, final Path p)
95    throws IOException {
96      if (fs.exists(p)) {
97        throw new IOException("File already exists " + p.toString());
98      }
99      if (!fs.createNewFile(p)) {
100       throw new IOException("Failed create of " + p);
101     }
102     return p;
103   }
104 
105   /**
106    * Checks to see if the specified file system is available
107    *
108    * @param fs filesystem
109    * @throws IOException e
110    */
111   public static void checkFileSystemAvailable(final FileSystem fs)
112   throws IOException {
113     if (!(fs instanceof DistributedFileSystem)) {
114       return;
115     }
116     IOException exception = null;
117     DistributedFileSystem dfs = (DistributedFileSystem) fs;
118     try {
119       if (dfs.exists(new Path("/"))) {
120         return;
121       }
122     } catch (IOException e) {
123       exception = RemoteExceptionHandler.checkIOException(e);
124     }
125     try {
126       fs.close();
127     } catch (Exception e) {
128         LOG.error("file system close failed: ", e);
129     }
130     IOException io = new IOException("File system is not available");
131     io.initCause(exception);
132     throw io;
133   }
134 
135   /**
136    * Verifies current version of file system
137    *
138    * @param fs filesystem object
139    * @param rootdir root hbase directory
140    * @return null if no version file exists, version string otherwise.
141    * @throws IOException e
142    */
143   public static String getVersion(FileSystem fs, Path rootdir)
144   throws IOException {
145     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
146     String version = null;
147     if (fs.exists(versionFile)) {
148       FSDataInputStream s =
149         fs.open(versionFile);
150       try {
151         version = DataInputStream.readUTF(s);
152       } finally {
153         s.close();
154       }
155     }
156     return version;
157   }
158 
159   /**
160    * Verifies current version of file system
161    *
162    * @param fs file system
163    * @param rootdir root directory of HBase installation
164    * @param message if true, issues a message on System.out
165    *
166    * @throws IOException e
167    */
168   public static void checkVersion(FileSystem fs, Path rootdir,
169       boolean message) throws IOException {
170     String version = getVersion(fs, rootdir);
171 
172     if (version == null) {
173       if (!rootRegionExists(fs, rootdir)) {
174         // rootDir is empty (no version file and no root region)
175         // just create new version file (HBASE-1195)
176         FSUtils.setVersion(fs, rootdir);
177         return;
178       }
179     } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0)
180         return;
181 
182     // version is deprecated require migration
183     // Output on stdout so user sees it in terminal.
184     String msg = "File system needs to be upgraded."
185       + "  You have version " + version
186       + " and I want version " + HConstants.FILE_SYSTEM_VERSION
187       + ".  Run the '${HBASE_HOME}/bin/hbase migrate' script.";
188     if (message) {
189       System.out.println("WARNING! " + msg);
190     }
191     throw new FileSystemVersionException(msg);
192   }
193 
194   /**
195    * Sets version of file system
196    *
197    * @param fs filesystem object
198    * @param rootdir hbase root
199    * @throws IOException e
200    */
201   public static void setVersion(FileSystem fs, Path rootdir)
202   throws IOException {
203     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION);
204   }
205 
206   /**
207    * Sets version of file system
208    *
209    * @param fs filesystem object
210    * @param rootdir hbase root directory
211    * @param version version to set
212    * @throws IOException e
213    */
214   public static void setVersion(FileSystem fs, Path rootdir, String version)
215   throws IOException {
216     FSDataOutputStream s =
217       fs.create(new Path(rootdir, HConstants.VERSION_FILE_NAME));
218     s.writeUTF(version);
219     s.close();
220     LOG.debug("Created version file at " + rootdir.toString() + " set its version at:" + version);
221   }
222 
223   /**
224    * Verifies root directory path is a valid URI with a scheme
225    *
226    * @param root root directory path
227    * @return Passed <code>root</code> argument.
228    * @throws IOException if not a valid URI with a scheme
229    */
230   public static Path validateRootPath(Path root) throws IOException {
231     try {
232       URI rootURI = new URI(root.toString());
233       String scheme = rootURI.getScheme();
234       if (scheme == null) {
235         throw new IOException("Root directory does not have a scheme");
236       }
237       return root;
238     } catch (URISyntaxException e) {
239       IOException io = new IOException("Root directory path is not a valid " +
240         "URI -- check your " + HConstants.HBASE_DIR + " configuration");
241       io.initCause(e);
242       throw io;
243     }
244   }
245 
246   /**
247    * If DFS, check safe mode and if so, wait until we clear it.
248    * @param conf configuration
249    * @param wait Sleep between retries
250    * @throws IOException e
251    */
252   public static void waitOnSafeMode(final Configuration conf,
253     final long wait)
254   throws IOException {
255     FileSystem fs = FileSystem.get(conf);
256     if (!(fs instanceof DistributedFileSystem)) return;
257     DistributedFileSystem dfs = (DistributedFileSystem)fs;
258     // Are there any data nodes up yet?
259     // Currently the safe mode check falls through if the namenode is up but no
260     // datanodes have reported in yet.
261     try {
262       while (dfs.getDataNodeStats().length == 0) {
263         LOG.info("Waiting for dfs to come up...");
264         try {
265           Thread.sleep(wait);
266         } catch (InterruptedException e) {
267           //continue
268         }
269       }
270     } catch (IOException e) {
271       // getDataNodeStats can fail if superuser privilege is required to run
272       // the datanode report, just ignore it
273     }
274     // Make sure dfs is not in safe mode
275     while (dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET)) {
276       LOG.info("Waiting for dfs to exit safe mode...");
277       try {
278         Thread.sleep(wait);
279       } catch (InterruptedException e) {
280         //continue
281       }
282     }
283   }
284 
285   /**
286    * Return the 'path' component of a Path.  In Hadoop, Path is an URI.  This
287    * method returns the 'path' component of a Path's URI: e.g. If a Path is
288    * <code>hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir</code>,
289    * this method returns <code>/hbase_trunk/TestTable/compaction.dir</code>.
290    * This method is useful if you want to print out a Path without qualifying
291    * Filesystem instance.
292    * @param p Filesystem Path whose 'path' component we are to return.
293    * @return Path portion of the Filesystem
294    */
295   public static String getPath(Path p) {
296     return p.toUri().getPath();
297   }
298 
299   /**
300    * @param c configuration
301    * @return Path to hbase root directory: i.e. <code>hbase.rootdir</code> from
302    * configuration as a Path.
303    * @throws IOException e
304    */
305   public static Path getRootDir(final Configuration c) throws IOException {
306     return new Path(c.get(HConstants.HBASE_DIR));
307   }
308 
309   /**
310    * Checks if root region exists
311    *
312    * @param fs file system
313    * @param rootdir root directory of HBase installation
314    * @return true if exists
315    * @throws IOException e
316    */
317   public static boolean rootRegionExists(FileSystem fs, Path rootdir)
318   throws IOException {
319     Path rootRegionDir =
320       HRegion.getRegionDir(rootdir, HRegionInfo.ROOT_REGIONINFO);
321     return fs.exists(rootRegionDir);
322   }
323 
324   /**
325    * Runs through the hbase rootdir and checks all stores have only
326    * one file in them -- that is, they've been major compacted.  Looks
327    * at root and meta tables too.
328    * @param fs filesystem
329    * @param hbaseRootDir hbase root directory
330    * @return True if this hbase install is major compacted.
331    * @throws IOException e
332    */
333   public static boolean isMajorCompacted(final FileSystem fs,
334       final Path hbaseRootDir)
335   throws IOException {
336     // Presumes any directory under hbase.rootdir is a table.
337     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs));
338     for (FileStatus tableDir : tableDirs) {
339       // Skip the .log directory.  All others should be tables.  Inside a table,
340       // there are compaction.dir directories to skip.  Otherwise, all else
341       // should be regions.  Then in each region, should only be family
342       // directories.  Under each of these, should be one file only.
343       Path d = tableDir.getPath();
344       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
345         continue;
346       }
347       FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
348       for (FileStatus regionDir : regionDirs) {
349         Path dd = regionDir.getPath();
350         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
351           continue;
352         }
353         // Else its a region name.  Now look in region for families.
354         FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
355         for (FileStatus familyDir : familyDirs) {
356           Path family = familyDir.getPath();
357           // Now in family make sure only one file.
358           FileStatus[] familyStatus = fs.listStatus(family);
359           if (familyStatus.length > 1) {
360             LOG.debug(family.toString() + " has " + familyStatus.length +
361                 " files.");
362             return false;
363           }
364         }
365       }
366     }
367     return true;
368   }
369 
370   // TODO move this method OUT of FSUtils. No dependencies to HMaster
371   /**
372    * Returns the total overall fragmentation percentage. Includes .META. and
373    * -ROOT- as well.
374    *
375    * @param master  The master defining the HBase root and file system.
376    * @return A map for each table and its percentage.
377    * @throws IOException When scanning the directory fails.
378    */
379   public static int getTotalTableFragmentation(final HMaster master)
380   throws IOException {
381     Map<String, Integer> map = getTableFragmentation(master);
382     return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
383   }
384 
385   /**
386    * Runs through the HBase rootdir and checks how many stores for each table
387    * have more than one file in them. Checks -ROOT- and .META. too. The total
388    * percentage across all tables is stored under the special key "-TOTAL-".
389    *
390    * @param master  The master defining the HBase root and file system.
391    * @return A map for each table and its percentage.
392    * @throws IOException When scanning the directory fails.
393    */
394   public static Map<String, Integer> getTableFragmentation(
395     final HMaster master)
396   throws IOException {
397     Path path = master.getRootDir();
398     // since HMaster.getFileSystem() is package private
399     FileSystem fs = path.getFileSystem(master.getConfiguration());
400     return getTableFragmentation(fs, path);
401   }
402 
403   /**
404    * Runs through the HBase rootdir and checks how many stores for each table
405    * have more than one file in them. Checks -ROOT- and .META. too. The total
406    * percentage across all tables is stored under the special key "-TOTAL-".
407    *
408    * @param fs  The file system to use.
409    * @param hbaseRootDir  The root directory to scan.
410    * @return A map for each table and its percentage.
411    * @throws IOException When scanning the directory fails.
412    */
413   public static Map<String, Integer> getTableFragmentation(
414     final FileSystem fs, final Path hbaseRootDir)
415   throws IOException {
416     Map<String, Integer> frags = new HashMap<String, Integer>();
417     int cfCountTotal = 0;
418     int cfFragTotal = 0;
419     DirFilter df = new DirFilter(fs);
420     // presumes any directory under hbase.rootdir is a table
421     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
422     for (FileStatus tableDir : tableDirs) {
423       // Skip the .log directory.  All others should be tables.  Inside a table,
424       // there are compaction.dir directories to skip.  Otherwise, all else
425       // should be regions.  Then in each region, should only be family
426       // directories.  Under each of these, should be one file only.
427       Path d = tableDir.getPath();
428       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
429         continue;
430       }
431       int cfCount = 0;
432       int cfFrag = 0;
433       FileStatus[] regionDirs = fs.listStatus(d, df);
434       for (FileStatus regionDir : regionDirs) {
435         Path dd = regionDir.getPath();
436         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
437           continue;
438         }
439         // else its a region name, now look in region for families
440         FileStatus[] familyDirs = fs.listStatus(dd, df);
441         for (FileStatus familyDir : familyDirs) {
442           cfCount++;
443           cfCountTotal++;
444           Path family = familyDir.getPath();
445           // now in family make sure only one file
446           FileStatus[] familyStatus = fs.listStatus(family);
447           if (familyStatus.length > 1) {
448             cfFrag++;
449             cfFragTotal++;
450           }
451         }
452       }
453       // compute percentage per table and store in result list
454       frags.put(d.getName(), Math.round((float) cfFrag / cfCount * 100));
455     }
456     // set overall percentage for all tables
457     frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100));
458     return frags;
459   }
460 
461   /**
462    * Expects to find -ROOT- directory.
463    * @param fs filesystem
464    * @param hbaseRootDir hbase root directory
465    * @return True if this a pre020 layout.
466    * @throws IOException e
467    */
468   public static boolean isPre020FileLayout(final FileSystem fs,
469     final Path hbaseRootDir)
470   throws IOException {
471     Path mapfiles = new Path(new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
472       "70236052"), "info"), "mapfiles");
473     return fs.exists(mapfiles);
474   }
475 
476   /**
477    * Runs through the hbase rootdir and checks all stores have only
478    * one file in them -- that is, they've been major compacted.  Looks
479    * at root and meta tables too.  This version differs from
480    * {@link #isMajorCompacted(FileSystem, Path)} in that it expects a
481    * pre-0.20.0 hbase layout on the filesystem.  Used migrating.
482    * @param fs filesystem
483    * @param hbaseRootDir hbase root directory
484    * @return True if this hbase install is major compacted.
485    * @throws IOException e
486    */
487   public static boolean isMajorCompactedPre020(final FileSystem fs,
488       final Path hbaseRootDir)
489   throws IOException {
490     // Presumes any directory under hbase.rootdir is a table.
491     FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs));
492     for (FileStatus tableDir : tableDirs) {
493       // Inside a table, there are compaction.dir directories to skip.
494       // Otherwise, all else should be regions.  Then in each region, should
495       // only be family directories.  Under each of these, should be a mapfile
496       // and info directory and in these only one file.
497       Path d = tableDir.getPath();
498       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
499         continue;
500       }
501       FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
502       for (FileStatus regionDir : regionDirs) {
503         Path dd = regionDir.getPath();
504         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
505           continue;
506         }
507         // Else its a region name.  Now look in region for families.
508         FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
509         for (FileStatus familyDir : familyDirs) {
510           Path family = familyDir.getPath();
511           FileStatus[] infoAndMapfile = fs.listStatus(family);
512           // Assert that only info and mapfile in family dir.
513           if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
514             LOG.debug(family.toString() +
515                 " has more than just info and mapfile: " + infoAndMapfile.length);
516             return false;
517           }
518           // Make sure directory named info or mapfile.
519           for (int ll = 0; ll < 2; ll++) {
520             if (infoAndMapfile[ll].getPath().getName().equals("info") ||
521                 infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
522               continue;
523             LOG.debug("Unexpected directory name: " +
524                 infoAndMapfile[ll].getPath());
525             return false;
526           }
527           // Now in family, there are 'mapfile' and 'info' subdirs.  Just
528           // look in the 'mapfile' subdir.
529           FileStatus[] familyStatus =
530               fs.listStatus(new Path(family, "mapfiles"));
531           if (familyStatus.length > 1) {
532             LOG.debug(family.toString() + " has " + familyStatus.length +
533                 " files.");
534             return false;
535           }
536         }
537       }
538     }
539     return true;
540   }
541 
542   /**
543    * A {@link PathFilter} that returns directories.
544    */
545   public static class DirFilter implements PathFilter {
546     private final FileSystem fs;
547 
548     public DirFilter(final FileSystem fs) {
549       this.fs = fs;
550     }
551 
552     public boolean accept(Path p) {
553       boolean isdir = false;
554       try {
555         isdir = this.fs.getFileStatus(p).isDir();
556       } catch (IOException e) {
557         e.printStackTrace();
558       }
559       return isdir;
560     }
561   }
562 
563   /**
564    * Heuristic to determine whether is safe or not to open a file for append
565    * Looks both for dfs.support.append and use reflection to search
566    * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
567    * @param conf
568    * @return True if append support
569    */
570   public static boolean isAppendSupported(final Configuration conf) {
571     boolean append = conf.getBoolean("dfs.support.append", false);
572     if (append) {
573       try {
574         // TODO: The implementation that comes back when we do a createWriter
575         // may not be using SequenceFile so the below is not a definitive test.
576         // Will do for now (hdfs-200).
577         SequenceFile.Writer.class.getMethod("syncFs", new Class<?> []{});
578         append = true;
579       } catch (SecurityException e) {
580       } catch (NoSuchMethodException e) {
581         append = false;
582       }
583     } else {
584       try {
585         FSDataOutputStream.class.getMethod("hflush", new Class<?> []{});
586       } catch (NoSuchMethodException e) {
587         append = false;
588       }
589     }
590     return append;
591   }
592 
593 
594   /*
595    * Recover file lease. Used when a file might be suspect to be had been left open by another process. <code>p</code>
596    * @param fs
597    * @param p
598    * @param append True if append supported
599    * @throws IOException
600    */
601   public static void recoverFileLease(final FileSystem fs, final Path p, Configuration conf)
602   throws IOException{
603     if (!isAppendSupported(conf)) {
604       LOG.warn("Running on HDFS without append enabled may result in data loss");
605       return;
606     }
607     // lease recovery not needed for local file system case.
608     // currently, local file system doesn't implement append either.
609     if (!(fs instanceof DistributedFileSystem)) {
610       return;
611     }
612     LOG.info("Recovering file" + p);
613     long startWaiting = System.currentTimeMillis();
614 
615     // Trying recovery
616     boolean recovered = false;
617     while (!recovered) {
618       try {
619         FSDataOutputStream out = fs.append(p);
620         out.close();
621         recovered = true;
622       } catch (IOException e) {
623         e = RemoteExceptionHandler.checkIOException(e);
624         if (e instanceof AlreadyBeingCreatedException) {
625           // We expect that we'll get this message while the lease is still
626           // within its soft limit, but if we get it past that, it means
627           // that the RS is holding onto the file even though it lost its
628           // znode. We could potentially abort after some time here.
629           long waitedFor = System.currentTimeMillis() - startWaiting;
630           if (waitedFor > FSConstants.LEASE_SOFTLIMIT_PERIOD) {
631             LOG.warn("Waited " + waitedFor + "ms for lease recovery on " + p +
632               ":" + e.getMessage());
633           }
634           try {
635             Thread.sleep(1000);
636           } catch (InterruptedException ex) {
637             // ignore it and try again
638           }
639         } else {
640           throw new IOException("Failed to open " + p + " for append", e);
641         }
642       }
643     }
644     LOG.info("Finished lease recover attempt for " + p);
645   }
646 
647 }