View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.DataInputStream;
23  import java.io.EOFException;
24  import java.io.FileNotFoundException;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.lang.reflect.InvocationTargetException;
28  import java.lang.reflect.Method;
29  import java.net.InetSocketAddress;
30  import java.net.URI;
31  import java.net.URISyntaxException;
32  import java.util.ArrayList;
33  import java.util.Collections;
34  import java.util.HashMap;
35  import java.util.LinkedList;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.concurrent.ArrayBlockingQueue;
39  import java.util.concurrent.ConcurrentHashMap;
40  import java.util.concurrent.ThreadPoolExecutor;
41  import java.util.concurrent.TimeUnit;
42  import java.util.regex.Pattern;
43  
44  import org.apache.commons.logging.Log;
45  import org.apache.commons.logging.LogFactory;
46  import org.apache.hadoop.classification.InterfaceAudience;
47  import org.apache.hadoop.conf.Configuration;
48  import org.apache.hadoop.fs.BlockLocation;
49  import org.apache.hadoop.fs.FSDataInputStream;
50  import org.apache.hadoop.fs.FSDataOutputStream;
51  import org.apache.hadoop.fs.FileStatus;
52  import org.apache.hadoop.fs.FileSystem;
53  import org.apache.hadoop.fs.Path;
54  import org.apache.hadoop.fs.PathFilter;
55  import org.apache.hadoop.fs.permission.FsAction;
56  import org.apache.hadoop.fs.permission.FsPermission;
57  import org.apache.hadoop.hbase.ClusterId;
58  import org.apache.hadoop.hbase.HColumnDescriptor;
59  import org.apache.hadoop.hbase.HConstants;
60  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
61  import org.apache.hadoop.hbase.HRegionInfo;
62  import org.apache.hadoop.hbase.RemoteExceptionHandler;
63  import org.apache.hadoop.hbase.TableName;
64  import org.apache.hadoop.hbase.exceptions.DeserializationException;
65  import org.apache.hadoop.hbase.fs.HFileSystem;
66  import org.apache.hadoop.hbase.master.HMaster;
67  import org.apache.hadoop.hbase.master.RegionPlacementMaintainer;
68  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
69  import org.apache.hadoop.hbase.protobuf.generated.FSProtos;
70  import org.apache.hadoop.hbase.regionserver.HRegion;
71  import org.apache.hadoop.hdfs.DistributedFileSystem;
72  import org.apache.hadoop.hdfs.protocol.FSConstants;
73  import org.apache.hadoop.io.IOUtils;
74  import org.apache.hadoop.io.SequenceFile;
75  import org.apache.hadoop.security.AccessControlException;
76  import org.apache.hadoop.security.UserGroupInformation;
77  import org.apache.hadoop.util.Progressable;
78  import org.apache.hadoop.util.ReflectionUtils;
79  import org.apache.hadoop.util.StringUtils;
80  
81  import com.google.common.primitives.Ints;
82  import com.google.protobuf.InvalidProtocolBufferException;
83  
84  /**
85   * Utility methods for interacting with the underlying file system.
86   */
87  @InterfaceAudience.Private
88  public abstract class FSUtils {
89    private static final Log LOG = LogFactory.getLog(FSUtils.class);
90  
91    /** Full access permissions (starting point for a umask) */
92    private static final String FULL_RWX_PERMISSIONS = "777";
93    private static final String THREAD_POOLSIZE = "hbase.client.localityCheck.threadPoolSize";
94    private static final int DEFAULT_THREAD_POOLSIZE = 2;
95  
96    /** Set to true on Windows platforms */
97    public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");
98  
99    protected FSUtils() {
100     super();
101   }
102 
103   /**
104    * Compare of path component. Does not consider schema; i.e. if schemas different but <code>path
105    * <code> starts with <code>rootPath<code>, then the function returns true
106    * @param rootPath
107    * @param path
108    * @return True if <code>path</code> starts with <code>rootPath</code>
109    */
110   public static boolean isStartingWithPath(final Path rootPath, final String path) {
111     String uriRootPath = rootPath.toUri().getPath();
112     String tailUriPath = (new Path(path)).toUri().getPath();
113     return tailUriPath.startsWith(uriRootPath);
114   }
115 
116   /**
117    * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
118    * '/a/b/c' part. Does not consider schema; i.e. if schemas different but path or subpath matches,
119    * the two will equate.
120    * @param pathToSearch Path we will be trying to match.
121    * @param pathTail
122    * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
123    */
124   public static boolean isMatchingTail(final Path pathToSearch, String pathTail) {
125     return isMatchingTail(pathToSearch, new Path(pathTail));
126   }
127 
128   /**
129    * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
130    * '/a/b/c' part. If you passed in 'hdfs://a/b/c and b/c, it would return true.  Does not consider
131    * schema; i.e. if schemas different but path or subpath matches, the two will equate.
132    * @param pathToSearch Path we will be trying to match.
133    * @param pathTail
134    * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
135    */
136   public static boolean isMatchingTail(final Path pathToSearch, final Path pathTail) {
137     if (pathToSearch.depth() != pathTail.depth()) return false;
138     Path tailPath = pathTail;
139     String tailName;
140     Path toSearch = pathToSearch;
141     String toSearchName;
142     boolean result = false;
143     do {
144       tailName = tailPath.getName();
145       if (tailName == null || tailName.length() <= 0) {
146         result = true;
147         break;
148       }
149       toSearchName = toSearch.getName();
150       if (toSearchName == null || toSearchName.length() <= 0) break;
151       // Move up a parent on each path for next go around.  Path doesn't let us go off the end.
152       tailPath = tailPath.getParent();
153       toSearch = toSearch.getParent();
154     } while(tailName.equals(toSearchName));
155     return result;
156   }
157 
158   public static FSUtils getInstance(FileSystem fs, Configuration conf) {
159     String scheme = fs.getUri().getScheme();
160     if (scheme == null) {
161       LOG.warn("Could not find scheme for uri " +
162           fs.getUri() + ", default to hdfs");
163       scheme = "hdfs";
164     }
165     Class<?> fsUtilsClass = conf.getClass("hbase.fsutil." +
166         scheme + ".impl", FSHDFSUtils.class); // Default to HDFS impl
167     FSUtils fsUtils = (FSUtils)ReflectionUtils.newInstance(fsUtilsClass, conf);
168     return fsUtils;
169   }
170 
171   /**
172    * Delete if exists.
173    * @param fs filesystem object
174    * @param dir directory to delete
175    * @return True if deleted <code>dir</code>
176    * @throws IOException e
177    */
178   public static boolean deleteDirectory(final FileSystem fs, final Path dir)
179   throws IOException {
180     return fs.exists(dir) && fs.delete(dir, true);
181   }
182 
183   /**
184    * Return the number of bytes that large input files should be optimally
185    * be split into to minimize i/o time.
186    *
187    * use reflection to search for getDefaultBlockSize(Path f)
188    * if the method doesn't exist, fall back to using getDefaultBlockSize()
189    *
190    * @param fs filesystem object
191    * @return the default block size for the path's filesystem
192    * @throws IOException e
193    */
194   public static long getDefaultBlockSize(final FileSystem fs, final Path path) throws IOException {
195     Method m = null;
196     Class<? extends FileSystem> cls = fs.getClass();
197     try {
198       m = cls.getMethod("getDefaultBlockSize", new Class<?>[] { Path.class });
199     } catch (NoSuchMethodException e) {
200       LOG.info("FileSystem doesn't support getDefaultBlockSize");
201     } catch (SecurityException e) {
202       LOG.info("Doesn't have access to getDefaultBlockSize on FileSystems", e);
203       m = null; // could happen on setAccessible()
204     }
205     if (m == null) {
206       return fs.getDefaultBlockSize();
207     } else {
208       try {
209         Object ret = m.invoke(fs, path);
210         return ((Long)ret).longValue();
211       } catch (Exception e) {
212         throw new IOException(e);
213       }
214     }
215   }
216 
217   /*
218    * Get the default replication.
219    *
220    * use reflection to search for getDefaultReplication(Path f)
221    * if the method doesn't exist, fall back to using getDefaultReplication()
222    *
223    * @param fs filesystem object
224    * @param f path of file
225    * @return default replication for the path's filesystem
226    * @throws IOException e
227    */
228   public static short getDefaultReplication(final FileSystem fs, final Path path) throws IOException {
229     Method m = null;
230     Class<? extends FileSystem> cls = fs.getClass();
231     try {
232       m = cls.getMethod("getDefaultReplication", new Class<?>[] { Path.class });
233     } catch (NoSuchMethodException e) {
234       LOG.info("FileSystem doesn't support getDefaultReplication");
235     } catch (SecurityException e) {
236       LOG.info("Doesn't have access to getDefaultReplication on FileSystems", e);
237       m = null; // could happen on setAccessible()
238     }
239     if (m == null) {
240       return fs.getDefaultReplication();
241     } else {
242       try {
243         Object ret = m.invoke(fs, path);
244         return ((Number)ret).shortValue();
245       } catch (Exception e) {
246         throw new IOException(e);
247       }
248     }
249   }
250 
251   /**
252    * Returns the default buffer size to use during writes.
253    *
254    * The size of the buffer should probably be a multiple of hardware
255    * page size (4096 on Intel x86), and it determines how much data is
256    * buffered during read and write operations.
257    *
258    * @param fs filesystem object
259    * @return default buffer size to use during writes
260    */
261   public static int getDefaultBufferSize(final FileSystem fs) {
262     return fs.getConf().getInt("io.file.buffer.size", 4096);
263   }
264 
265   /**
266    * Create the specified file on the filesystem. By default, this will:
267    * <ol>
268    * <li>overwrite the file if it exists</li>
269    * <li>apply the umask in the configuration (if it is enabled)</li>
270    * <li>use the fs configured buffer size (or 4096 if not set)</li>
271    * <li>use the default replication</li>
272    * <li>use the default block size</li>
273    * <li>not track progress</li>
274    * </ol>
275    *
276    * @param fs {@link FileSystem} on which to write the file
277    * @param path {@link Path} to the file to write
278    * @param perm permissions
279    * @param favoredNodes
280    * @return output stream to the created file
281    * @throws IOException if the file cannot be created
282    */
283   public static FSDataOutputStream create(FileSystem fs, Path path,
284       FsPermission perm, InetSocketAddress[] favoredNodes) throws IOException {
285     if (fs instanceof HFileSystem) {
286       FileSystem backingFs = ((HFileSystem)fs).getBackingFs();
287       if (backingFs instanceof DistributedFileSystem) {
288         // Try to use the favoredNodes version via reflection to allow backwards-
289         // compatibility.
290         try {
291           return (FSDataOutputStream) (DistributedFileSystem.class
292               .getDeclaredMethod("create", Path.class, FsPermission.class,
293                   boolean.class, int.class, short.class, long.class,
294                   Progressable.class, InetSocketAddress[].class)
295                   .invoke(backingFs, path, FsPermission.getDefault(), true,
296                       getDefaultBufferSize(backingFs),
297                       getDefaultReplication(backingFs, path),
298                       getDefaultBlockSize(backingFs, path),
299                       null, favoredNodes));
300         } catch (InvocationTargetException ite) {
301           // Function was properly called, but threw it's own exception.
302           throw new IOException(ite.getCause());
303         } catch (NoSuchMethodException e) {
304           LOG.debug("DFS Client does not support most favored nodes create; using default create");
305           if (LOG.isTraceEnabled()) LOG.trace("Ignoring; use default create", e);
306         } catch (IllegalArgumentException e) {
307           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
308         } catch (SecurityException e) {
309           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
310         } catch (IllegalAccessException e) {
311           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
312         }
313       }
314     }
315     return create(fs, path, perm, true);
316   }
317 
318   /**
319    * Create the specified file on the filesystem. By default, this will:
320    * <ol>
321    * <li>apply the umask in the configuration (if it is enabled)</li>
322    * <li>use the fs configured buffer size (or 4096 if not set)</li>
323    * <li>use the default replication</li>
324    * <li>use the default block size</li>
325    * <li>not track progress</li>
326    * </ol>
327    *
328    * @param fs {@link FileSystem} on which to write the file
329    * @param path {@link Path} to the file to write
330    * @param perm
331    * @param overwrite Whether or not the created file should be overwritten.
332    * @return output stream to the created file
333    * @throws IOException if the file cannot be created
334    */
335   public static FSDataOutputStream create(FileSystem fs, Path path,
336       FsPermission perm, boolean overwrite) throws IOException {
337     if (LOG.isTraceEnabled()) {
338       LOG.trace("Creating file=" + path + " with permission=" + perm + ", overwrite=" + overwrite);
339     }
340     return fs.create(path, perm, overwrite, getDefaultBufferSize(fs),
341         getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null);
342   }
343 
344   /**
345    * Get the file permissions specified in the configuration, if they are
346    * enabled.
347    *
348    * @param fs filesystem that the file will be created on.
349    * @param conf configuration to read for determining if permissions are
350    *          enabled and which to use
351    * @param permssionConfKey property key in the configuration to use when
352    *          finding the permission
353    * @return the permission to use when creating a new file on the fs. If
354    *         special permissions are not specified in the configuration, then
355    *         the default permissions on the the fs will be returned.
356    */
357   public static FsPermission getFilePermissions(final FileSystem fs,
358       final Configuration conf, final String permssionConfKey) {
359     boolean enablePermissions = conf.getBoolean(
360         HConstants.ENABLE_DATA_FILE_UMASK, false);
361 
362     if (enablePermissions) {
363       try {
364         FsPermission perm = new FsPermission(FULL_RWX_PERMISSIONS);
365         // make sure that we have a mask, if not, go default.
366         String mask = conf.get(permssionConfKey);
367         if (mask == null)
368           return FsPermission.getDefault();
369         // appy the umask
370         FsPermission umask = new FsPermission(mask);
371         return perm.applyUMask(umask);
372       } catch (IllegalArgumentException e) {
373         LOG.warn(
374             "Incorrect umask attempted to be created: "
375                 + conf.get(permssionConfKey)
376                 + ", using default file permissions.", e);
377         return FsPermission.getDefault();
378       }
379     }
380     return FsPermission.getDefault();
381   }
382 
383   /**
384    * Checks to see if the specified file system is available
385    *
386    * @param fs filesystem
387    * @throws IOException e
388    */
389   public static void checkFileSystemAvailable(final FileSystem fs)
390   throws IOException {
391     if (!(fs instanceof DistributedFileSystem)) {
392       return;
393     }
394     IOException exception = null;
395     DistributedFileSystem dfs = (DistributedFileSystem) fs;
396     try {
397       if (dfs.exists(new Path("/"))) {
398         return;
399       }
400     } catch (IOException e) {
401       exception = RemoteExceptionHandler.checkIOException(e);
402     }
403     try {
404       fs.close();
405     } catch (Exception e) {
406       LOG.error("file system close failed: ", e);
407     }
408     IOException io = new IOException("File system is not available");
409     io.initCause(exception);
410     throw io;
411   }
412 
413   /**
414    * We use reflection because {@link DistributedFileSystem#setSafeMode(
415    * FSConstants.SafeModeAction action, boolean isChecked)} is not in hadoop 1.1
416    *
417    * @param dfs
418    * @return whether we're in safe mode
419    * @throws IOException
420    */
421   private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {
422     boolean inSafeMode = false;
423     try {
424       Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class<?> []{
425           org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class});
426       inSafeMode = (Boolean) m.invoke(dfs,
427         org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true);
428     } catch (Exception e) {
429       if (e instanceof IOException) throw (IOException) e;
430 
431       // Check whether dfs is on safemode.
432       inSafeMode = dfs.setSafeMode(
433         org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET);
434     }
435     return inSafeMode;
436   }
437 
438   /**
439    * Check whether dfs is in safemode.
440    * @param conf
441    * @throws IOException
442    */
443   public static void checkDfsSafeMode(final Configuration conf)
444   throws IOException {
445     boolean isInSafeMode = false;
446     FileSystem fs = FileSystem.get(conf);
447     if (fs instanceof DistributedFileSystem) {
448       DistributedFileSystem dfs = (DistributedFileSystem)fs;
449       isInSafeMode = isInSafeMode(dfs);
450     }
451     if (isInSafeMode) {
452       throw new IOException("File system is in safemode, it can't be written now");
453     }
454   }
455 
456   /**
457    * Verifies current version of file system
458    *
459    * @param fs filesystem object
460    * @param rootdir root hbase directory
461    * @return null if no version file exists, version string otherwise.
462    * @throws IOException e
463    * @throws org.apache.hadoop.hbase.exceptions.DeserializationException
464    */
465   public static String getVersion(FileSystem fs, Path rootdir)
466   throws IOException, DeserializationException {
467     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
468     FileStatus[] status = null;
469     try {
470       // hadoop 2.0 throws FNFE if directory does not exist.
471       // hadoop 1.0 returns null if directory does not exist.
472       status = fs.listStatus(versionFile);
473     } catch (FileNotFoundException fnfe) {
474       return null;
475     }
476     if (status == null || status.length == 0) return null;
477     String version = null;
478     byte [] content = new byte [(int)status[0].getLen()];
479     FSDataInputStream s = fs.open(versionFile);
480     try {
481       IOUtils.readFully(s, content, 0, content.length);
482       if (ProtobufUtil.isPBMagicPrefix(content)) {
483         version = parseVersionFrom(content);
484       } else {
485         // Presume it pre-pb format.
486         InputStream is = new ByteArrayInputStream(content);
487         DataInputStream dis = new DataInputStream(is);
488         try {
489           version = dis.readUTF();
490         } finally {
491           dis.close();
492         }
493         // Update the format
494         LOG.info("Updating the hbase.version file format with version=" + version);
495         setVersion(fs, rootdir, version, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
496       }
497     } catch (EOFException eof) {
498       LOG.warn("Version file was empty, odd, will try to set it.");
499     } finally {
500       s.close();
501     }
502     return version;
503   }
504 
505   /**
506    * Parse the content of the ${HBASE_ROOTDIR}/hbase.version file.
507    * @param bytes The byte content of the hbase.version file.
508    * @return The version found in the file as a String.
509    * @throws DeserializationException
510    */
511   static String parseVersionFrom(final byte [] bytes)
512   throws DeserializationException {
513     ProtobufUtil.expectPBMagicPrefix(bytes);
514     int pblen = ProtobufUtil.lengthOfPBMagic();
515     FSProtos.HBaseVersionFileContent.Builder builder =
516       FSProtos.HBaseVersionFileContent.newBuilder();
517     FSProtos.HBaseVersionFileContent fileContent;
518     try {
519       fileContent = builder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
520       return fileContent.getVersion();
521     } catch (InvalidProtocolBufferException e) {
522       // Convert
523       throw new DeserializationException(e);
524     }
525   }
526 
527   /**
528    * Create the content to write into the ${HBASE_ROOTDIR}/hbase.version file.
529    * @param version Version to persist
530    * @return Serialized protobuf with <code>version</code> content and a bit of pb magic for a prefix.
531    */
532   static byte [] toVersionByteArray(final String version) {
533     FSProtos.HBaseVersionFileContent.Builder builder =
534       FSProtos.HBaseVersionFileContent.newBuilder();
535     return ProtobufUtil.prependPBMagic(builder.setVersion(version).build().toByteArray());
536   }
537 
538   /**
539    * Verifies current version of file system
540    *
541    * @param fs file system
542    * @param rootdir root directory of HBase installation
543    * @param message if true, issues a message on System.out
544    *
545    * @throws IOException e
546    * @throws DeserializationException
547    */
548   public static void checkVersion(FileSystem fs, Path rootdir, boolean message)
549   throws IOException, DeserializationException {
550     checkVersion(fs, rootdir, message, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
551   }
552 
553   /**
554    * Verifies current version of file system
555    *
556    * @param fs file system
557    * @param rootdir root directory of HBase installation
558    * @param message if true, issues a message on System.out
559    * @param wait wait interval
560    * @param retries number of times to retry
561    *
562    * @throws IOException e
563    * @throws DeserializationException
564    */
565   public static void checkVersion(FileSystem fs, Path rootdir,
566       boolean message, int wait, int retries)
567   throws IOException, DeserializationException {
568     String version = getVersion(fs, rootdir);
569     if (version == null) {
570       if (!metaRegionExists(fs, rootdir)) {
571         // rootDir is empty (no version file and no root region)
572         // just create new version file (HBASE-1195)
573         setVersion(fs, rootdir, wait, retries);
574         return;
575       }
576     } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) return;
577 
578     // version is deprecated require migration
579     // Output on stdout so user sees it in terminal.
580     String msg = "HBase file layout needs to be upgraded."
581       + "  You have version " + version
582       + " and I want version " + HConstants.FILE_SYSTEM_VERSION
583       + ".  Is your hbase.rootdir valid?  If so, you may need to run "
584       + "'hbase hbck -fixVersionFile'.";
585     if (message) {
586       System.out.println("WARNING! " + msg);
587     }
588     throw new FileSystemVersionException(msg);
589   }
590 
591   /**
592    * Sets version of file system
593    *
594    * @param fs filesystem object
595    * @param rootdir hbase root
596    * @throws IOException e
597    */
598   public static void setVersion(FileSystem fs, Path rootdir)
599   throws IOException {
600     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,
601       HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
602   }
603 
604   /**
605    * Sets version of file system
606    *
607    * @param fs filesystem object
608    * @param rootdir hbase root
609    * @param wait time to wait for retry
610    * @param retries number of times to retry before failing
611    * @throws IOException e
612    */
613   public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries)
614   throws IOException {
615     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);
616   }
617 
618 
619   /**
620    * Sets version of file system
621    *
622    * @param fs filesystem object
623    * @param rootdir hbase root directory
624    * @param version version to set
625    * @param wait time to wait for retry
626    * @param retries number of times to retry before throwing an IOException
627    * @throws IOException e
628    */
629   public static void setVersion(FileSystem fs, Path rootdir, String version,
630       int wait, int retries) throws IOException {
631     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
632     while (true) {
633       try {
634         FSDataOutputStream s = fs.create(versionFile);
635         s.write(toVersionByteArray(version));
636         s.close();
637         LOG.debug("Created version file at " + rootdir.toString() + " with version=" + version);
638         return;
639       } catch (IOException e) {
640         if (retries > 0) {
641           LOG.warn("Unable to create version file at " + rootdir.toString() + ", retrying", e);
642           fs.delete(versionFile, false);
643           try {
644             if (wait > 0) {
645               Thread.sleep(wait);
646             }
647           } catch (InterruptedException ex) {
648             // ignore
649           }
650           retries--;
651         } else {
652           throw e;
653         }
654       }
655     }
656   }
657 
658   /**
659    * Checks that a cluster ID file exists in the HBase root directory
660    * @param fs the root directory FileSystem
661    * @param rootdir the HBase root directory in HDFS
662    * @param wait how long to wait between retries
663    * @return <code>true</code> if the file exists, otherwise <code>false</code>
664    * @throws IOException if checking the FileSystem fails
665    */
666   public static boolean checkClusterIdExists(FileSystem fs, Path rootdir,
667       int wait) throws IOException {
668     while (true) {
669       try {
670         Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
671         return fs.exists(filePath);
672       } catch (IOException ioe) {
673         if (wait > 0) {
674           LOG.warn("Unable to check cluster ID file in " + rootdir.toString() +
675               ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe));
676           try {
677             Thread.sleep(wait);
678           } catch (InterruptedException ie) {
679             Thread.interrupted();
680             break;
681           }
682         } else {
683           throw ioe;
684         }
685       }
686     }
687     return false;
688   }
689 
690   /**
691    * Returns the value of the unique cluster ID stored for this HBase instance.
692    * @param fs the root directory FileSystem
693    * @param rootdir the path to the HBase root directory
694    * @return the unique cluster identifier
695    * @throws IOException if reading the cluster ID file fails
696    */
697   public static ClusterId getClusterId(FileSystem fs, Path rootdir)
698   throws IOException {
699     Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
700     ClusterId clusterId = null;
701     FileStatus status = fs.exists(idPath)? fs.getFileStatus(idPath):  null;
702     if (status != null) {
703       int len = Ints.checkedCast(status.getLen());
704       byte [] content = new byte[len];
705       FSDataInputStream in = fs.open(idPath);
706       try {
707         in.readFully(content);
708       } catch (EOFException eof) {
709         LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
710       } finally{
711         in.close();
712       }
713       try {
714         clusterId = ClusterId.parseFrom(content);
715       } catch (DeserializationException e) {
716         throw new IOException("content=" + Bytes.toString(content), e);
717       }
718       // If not pb'd, make it so.
719       if (!ProtobufUtil.isPBMagicPrefix(content)) {
720         String cid = new String();
721         in = fs.open(idPath);
722         try {
723           cid = in.readUTF();
724           clusterId = new ClusterId(cid);
725         } catch (EOFException eof) {
726           LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
727         } finally {
728           in.close();
729         }
730         rewriteAsPb(fs, rootdir, idPath, clusterId);
731       }
732       return clusterId;
733     } else {
734       LOG.warn("Cluster ID file does not exist at " + idPath.toString());
735     }
736     return clusterId;
737   }
738 
739   /**
740    * @param cid
741    * @throws IOException
742    */
743   private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p,
744       final ClusterId cid)
745   throws IOException {
746     // Rewrite the file as pb.  Move aside the old one first, write new
747     // then delete the moved-aside file.
748     Path movedAsideName = new Path(p + "." + System.currentTimeMillis());
749     if (!fs.rename(p, movedAsideName)) throw new IOException("Failed rename of " + p);
750     setClusterId(fs, rootdir, cid, 100);
751     if (!fs.delete(movedAsideName, false)) {
752       throw new IOException("Failed delete of " + movedAsideName);
753     }
754     LOG.debug("Rewrote the hbase.id file as pb");
755   }
756 
757   /**
758    * Writes a new unique identifier for this cluster to the "hbase.id" file
759    * in the HBase root directory
760    * @param fs the root directory FileSystem
761    * @param rootdir the path to the HBase root directory
762    * @param clusterId the unique identifier to store
763    * @param wait how long (in milliseconds) to wait between retries
764    * @throws IOException if writing to the FileSystem fails and no wait value
765    */
766   public static void setClusterId(FileSystem fs, Path rootdir, ClusterId clusterId,
767       int wait) throws IOException {
768     while (true) {
769       try {
770         Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
771         FSDataOutputStream s = fs.create(filePath);
772         try {
773           s.write(clusterId.toByteArray());
774         } finally {
775           s.close();
776         }
777         if (LOG.isDebugEnabled()) {
778           LOG.debug("Created cluster ID file at " + filePath.toString() + " with ID: " + clusterId);
779         }
780         return;
781       } catch (IOException ioe) {
782         if (wait > 0) {
783           LOG.warn("Unable to create cluster ID file in " + rootdir.toString() +
784               ", retrying in " + wait + "msec: " + StringUtils.stringifyException(ioe));
785           try {
786             Thread.sleep(wait);
787           } catch (InterruptedException ie) {
788             Thread.interrupted();
789             break;
790           }
791         } else {
792           throw ioe;
793         }
794       }
795     }
796   }
797 
798   /**
799    * Verifies root directory path is a valid URI with a scheme
800    *
801    * @param root root directory path
802    * @return Passed <code>root</code> argument.
803    * @throws IOException if not a valid URI with a scheme
804    */
805   public static Path validateRootPath(Path root) throws IOException {
806     try {
807       URI rootURI = new URI(root.toString());
808       String scheme = rootURI.getScheme();
809       if (scheme == null) {
810         throw new IOException("Root directory does not have a scheme");
811       }
812       return root;
813     } catch (URISyntaxException e) {
814       IOException io = new IOException("Root directory path is not a valid " +
815         "URI -- check your " + HConstants.HBASE_DIR + " configuration");
816       io.initCause(e);
817       throw io;
818     }
819   }
820 
821   /**
822    * Checks for the presence of the root path (using the provided conf object) in the given path. If
823    * it exists, this method removes it and returns the String representation of remaining relative path.
824    * @param path
825    * @param conf
826    * @return String representation of the remaining relative path
827    * @throws IOException
828    */
829   public static String removeRootPath(Path path, final Configuration conf) throws IOException {
830     Path root = FSUtils.getRootDir(conf);
831     String pathStr = path.toString();
832     // check that the path is absolute... it has the root path in it.
833     if (!pathStr.startsWith(root.toString())) return pathStr;
834     // if not, return as it is.
835     return pathStr.substring(root.toString().length() + 1);// remove the "/" too.
836   }
837 
838   /**
839    * If DFS, check safe mode and if so, wait until we clear it.
840    * @param conf configuration
841    * @param wait Sleep between retries
842    * @throws IOException e
843    */
844   public static void waitOnSafeMode(final Configuration conf,
845     final long wait)
846   throws IOException {
847     FileSystem fs = FileSystem.get(conf);
848     if (!(fs instanceof DistributedFileSystem)) return;
849     DistributedFileSystem dfs = (DistributedFileSystem)fs;
850     // Make sure dfs is not in safe mode
851     while (isInSafeMode(dfs)) {
852       LOG.info("Waiting for dfs to exit safe mode...");
853       try {
854         Thread.sleep(wait);
855       } catch (InterruptedException e) {
856         //continue
857       }
858     }
859   }
860 
861   /**
862    * Return the 'path' component of a Path.  In Hadoop, Path is an URI.  This
863    * method returns the 'path' component of a Path's URI: e.g. If a Path is
864    * <code>hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir</code>,
865    * this method returns <code>/hbase_trunk/TestTable/compaction.dir</code>.
866    * This method is useful if you want to print out a Path without qualifying
867    * Filesystem instance.
868    * @param p Filesystem Path whose 'path' component we are to return.
869    * @return Path portion of the Filesystem
870    */
871   public static String getPath(Path p) {
872     return p.toUri().getPath();
873   }
874 
875   /**
876    * @param c configuration
877    * @return Path to hbase root directory: i.e. <code>hbase.rootdir</code> from
878    * configuration as a qualified Path.
879    * @throws IOException e
880    */
881   public static Path getRootDir(final Configuration c) throws IOException {
882     Path p = new Path(c.get(HConstants.HBASE_DIR));
883     FileSystem fs = p.getFileSystem(c);
884     return p.makeQualified(fs);
885   }
886 
887   public static void setRootDir(final Configuration c, final Path root) throws IOException {
888     c.set(HConstants.HBASE_DIR, root.toString());
889   }
890 
891   public static void setFsDefault(final Configuration c, final Path root) throws IOException {
892     c.set("fs.defaultFS", root.toString());    // for hadoop 0.21+
893     c.set("fs.default.name", root.toString()); // for hadoop 0.20
894   }
895 
896   /**
897    * Checks if meta region exists
898    *
899    * @param fs file system
900    * @param rootdir root directory of HBase installation
901    * @return true if exists
902    * @throws IOException e
903    */
904   @SuppressWarnings("deprecation")
905   public static boolean metaRegionExists(FileSystem fs, Path rootdir)
906   throws IOException {
907     Path metaRegionDir =
908       HRegion.getRegionDir(rootdir, HRegionInfo.FIRST_META_REGIONINFO);
909     return fs.exists(metaRegionDir);
910   }
911 
912   /**
913    * Compute HDFS blocks distribution of a given file, or a portion of the file
914    * @param fs file system
915    * @param status file status of the file
916    * @param start start position of the portion
917    * @param length length of the portion
918    * @return The HDFS blocks distribution
919    */
920   static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
921     final FileSystem fs, FileStatus status, long start, long length)
922     throws IOException {
923     HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
924     BlockLocation [] blockLocations =
925       fs.getFileBlockLocations(status, start, length);
926     for(BlockLocation bl : blockLocations) {
927       String [] hosts = bl.getHosts();
928       long len = bl.getLength();
929       blocksDistribution.addHostsAndBlockWeight(hosts, len);
930     }
931 
932     return blocksDistribution;
933   }
934 
935 
936 
937   /**
938    * Runs through the hbase rootdir and checks all stores have only
939    * one file in them -- that is, they've been major compacted.  Looks
940    * at root and meta tables too.
941    * @param fs filesystem
942    * @param hbaseRootDir hbase root directory
943    * @return True if this hbase install is major compacted.
944    * @throws IOException e
945    */
946   public static boolean isMajorCompacted(final FileSystem fs,
947       final Path hbaseRootDir)
948   throws IOException {
949     List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
950     for (Path d : tableDirs) {
951       FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
952       for (FileStatus regionDir : regionDirs) {
953         Path dd = regionDir.getPath();
954         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
955           continue;
956         }
957         // Else its a region name.  Now look in region for families.
958         FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
959         for (FileStatus familyDir : familyDirs) {
960           Path family = familyDir.getPath();
961           // Now in family make sure only one file.
962           FileStatus[] familyStatus = fs.listStatus(family);
963           if (familyStatus.length > 1) {
964             LOG.debug(family.toString() + " has " + familyStatus.length +
965                 " files.");
966             return false;
967           }
968         }
969       }
970     }
971     return true;
972   }
973 
974   // TODO move this method OUT of FSUtils. No dependencies to HMaster
975   /**
976    * Returns the total overall fragmentation percentage. Includes hbase:meta and
977    * -ROOT- as well.
978    *
979    * @param master  The master defining the HBase root and file system.
980    * @return A map for each table and its percentage.
981    * @throws IOException When scanning the directory fails.
982    */
983   public static int getTotalTableFragmentation(final HMaster master)
984   throws IOException {
985     Map<String, Integer> map = getTableFragmentation(master);
986     return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
987   }
988 
989   /**
990    * Runs through the HBase rootdir and checks how many stores for each table
991    * have more than one file in them. Checks -ROOT- and hbase:meta too. The total
992    * percentage across all tables is stored under the special key "-TOTAL-".
993    *
994    * @param master  The master defining the HBase root and file system.
995    * @return A map for each table and its percentage.
996    *
997    * @throws IOException When scanning the directory fails.
998    */
999   public static Map<String, Integer> getTableFragmentation(
1000     final HMaster master)
1001   throws IOException {
1002     Path path = getRootDir(master.getConfiguration());
1003     // since HMaster.getFileSystem() is package private
1004     FileSystem fs = path.getFileSystem(master.getConfiguration());
1005     return getTableFragmentation(fs, path);
1006   }
1007 
1008   /**
1009    * Runs through the HBase rootdir and checks how many stores for each table
1010    * have more than one file in them. Checks -ROOT- and hbase:meta too. The total
1011    * percentage across all tables is stored under the special key "-TOTAL-".
1012    *
1013    * @param fs  The file system to use.
1014    * @param hbaseRootDir  The root directory to scan.
1015    * @return A map for each table and its percentage.
1016    * @throws IOException When scanning the directory fails.
1017    */
1018   public static Map<String, Integer> getTableFragmentation(
1019     final FileSystem fs, final Path hbaseRootDir)
1020   throws IOException {
1021     Map<String, Integer> frags = new HashMap<String, Integer>();
1022     int cfCountTotal = 0;
1023     int cfFragTotal = 0;
1024     DirFilter df = new DirFilter(fs);
1025     List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
1026     for (Path d : tableDirs) {
1027       int cfCount = 0;
1028       int cfFrag = 0;
1029       FileStatus[] regionDirs = fs.listStatus(d, df);
1030       for (FileStatus regionDir : regionDirs) {
1031         Path dd = regionDir.getPath();
1032         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
1033           continue;
1034         }
1035         // else its a region name, now look in region for families
1036         FileStatus[] familyDirs = fs.listStatus(dd, df);
1037         for (FileStatus familyDir : familyDirs) {
1038           cfCount++;
1039           cfCountTotal++;
1040           Path family = familyDir.getPath();
1041           // now in family make sure only one file
1042           FileStatus[] familyStatus = fs.listStatus(family);
1043           if (familyStatus.length > 1) {
1044             cfFrag++;
1045             cfFragTotal++;
1046           }
1047         }
1048       }
1049       // compute percentage per table and store in result list
1050       frags.put(FSUtils.getTableName(d).getNameAsString(),
1051           Math.round((float) cfFrag / cfCount * 100));
1052     }
1053     // set overall percentage for all tables
1054     frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100));
1055     return frags;
1056   }
1057 
1058   /**
1059    * Expects to find -ROOT- directory.
1060    * @param fs filesystem
1061    * @param hbaseRootDir hbase root directory
1062    * @return True if this a pre020 layout.
1063    * @throws IOException e
1064    */
1065   public static boolean isPre020FileLayout(final FileSystem fs,
1066     final Path hbaseRootDir)
1067   throws IOException {
1068     Path mapfiles = new Path(new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
1069       "70236052"), "info"), "mapfiles");
1070     return fs.exists(mapfiles);
1071   }
1072 
1073   /**
1074    * Runs through the hbase rootdir and checks all stores have only
1075    * one file in them -- that is, they've been major compacted.  Looks
1076    * at root and meta tables too.  This version differs from
1077    * {@link #isMajorCompacted(FileSystem, Path)} in that it expects a
1078    * pre-0.20.0 hbase layout on the filesystem.  Used migrating.
1079    * @param fs filesystem
1080    * @param hbaseRootDir hbase root directory
1081    * @return True if this hbase install is major compacted.
1082    * @throws IOException e
1083    */
1084   public static boolean isMajorCompactedPre020(final FileSystem fs,
1085       final Path hbaseRootDir)
1086   throws IOException {
1087     // Presumes any directory under hbase.rootdir is a table.
1088     List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
1089     for (Path d: tableDirs) {
1090       // Inside a table, there are compaction.dir directories to skip.
1091       // Otherwise, all else should be regions.  Then in each region, should
1092       // only be family directories.  Under each of these, should be a mapfile
1093       // and info directory and in these only one file.
1094       if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
1095         continue;
1096       }
1097       FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
1098       for (FileStatus regionDir : regionDirs) {
1099         Path dd = regionDir.getPath();
1100         if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
1101           continue;
1102         }
1103         // Else its a region name.  Now look in region for families.
1104         FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
1105         for (FileStatus familyDir : familyDirs) {
1106           Path family = familyDir.getPath();
1107           FileStatus[] infoAndMapfile = fs.listStatus(family);
1108           // Assert that only info and mapfile in family dir.
1109           if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
1110             LOG.debug(family.toString() +
1111                 " has more than just info and mapfile: " + infoAndMapfile.length);
1112             return false;
1113           }
1114           // Make sure directory named info or mapfile.
1115           for (int ll = 0; ll < 2; ll++) {
1116             if (infoAndMapfile[ll].getPath().getName().equals("info") ||
1117                 infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
1118               continue;
1119             LOG.debug("Unexpected directory name: " +
1120                 infoAndMapfile[ll].getPath());
1121             return false;
1122           }
1123           // Now in family, there are 'mapfile' and 'info' subdirs.  Just
1124           // look in the 'mapfile' subdir.
1125           FileStatus[] familyStatus =
1126               fs.listStatus(new Path(family, "mapfiles"));
1127           if (familyStatus.length > 1) {
1128             LOG.debug(family.toString() + " has " + familyStatus.length +
1129                 " files.");
1130             return false;
1131           }
1132         }
1133       }
1134     }
1135     return true;
1136   }
1137 
1138   /**
1139    * Returns the {@link org.apache.hadoop.fs.Path} object representing the table directory under
1140    * path rootdir
1141    *
1142    * @param rootdir qualified path of HBase root directory
1143    * @param tableName name of table
1144    * @return {@link org.apache.hadoop.fs.Path} for table
1145    */
1146   public static Path getTableDir(Path rootdir, final TableName tableName) {
1147     return new Path(getNamespaceDir(rootdir, tableName.getNamespaceAsString()),
1148         tableName.getQualifierAsString());
1149   }
1150 
1151   /**
1152    * Returns the {@link org.apache.hadoop.hbase.TableName} object representing
1153    * the table directory under
1154    * path rootdir
1155    *
1156    * @param tablePath path of table
1157    * @return {@link org.apache.hadoop.fs.Path} for table
1158    */
1159   public static TableName getTableName(Path tablePath) {
1160     return TableName.valueOf(tablePath.getParent().getName(), tablePath.getName());
1161   }
1162 
1163   /**
1164    * Returns the {@link org.apache.hadoop.fs.Path} object representing
1165    * the namespace directory under path rootdir
1166    *
1167    * @param rootdir qualified path of HBase root directory
1168    * @param namespace namespace name
1169    * @return {@link org.apache.hadoop.fs.Path} for table
1170    */
1171   public static Path getNamespaceDir(Path rootdir, final String namespace) {
1172     return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR,
1173         new Path(namespace)));
1174   }
1175 
1176   /**
1177    * A {@link PathFilter} that returns only regular files.
1178    */
1179   static class FileFilter implements PathFilter {
1180     private final FileSystem fs;
1181 
1182     public FileFilter(final FileSystem fs) {
1183       this.fs = fs;
1184     }
1185 
1186     @Override
1187     public boolean accept(Path p) {
1188       try {
1189         return fs.isFile(p);
1190       } catch (IOException e) {
1191         LOG.debug("unable to verify if path=" + p + " is a regular file", e);
1192         return false;
1193       }
1194     }
1195   }
1196 
1197   /**
1198    * Directory filter that doesn't include any of the directories in the specified blacklist
1199    */
1200   public static class BlackListDirFilter implements PathFilter {
1201     private final FileSystem fs;
1202     private List<String> blacklist;
1203 
1204     /**
1205      * Create a filter on the give filesystem with the specified blacklist
1206      * @param fs filesystem to filter
1207      * @param directoryNameBlackList list of the names of the directories to filter. If
1208      *          <tt>null</tt>, all directories are returned
1209      */
1210     @SuppressWarnings("unchecked")
1211     public BlackListDirFilter(final FileSystem fs, final List<String> directoryNameBlackList) {
1212       this.fs = fs;
1213       blacklist =
1214         (List<String>) (directoryNameBlackList == null ? Collections.emptyList()
1215           : directoryNameBlackList);
1216     }
1217 
1218     @Override
1219     public boolean accept(Path p) {
1220       boolean isValid = false;
1221       try {
1222         if (blacklist.contains(p.getName().toString())) {
1223           isValid = false;
1224         } else {
1225           isValid = fs.getFileStatus(p).isDir();
1226         }
1227       } catch (IOException e) {
1228         LOG.warn("An error occurred while verifying if [" + p.toString()
1229             + "] is a valid directory. Returning 'not valid' and continuing.", e);
1230       }
1231       return isValid;
1232     }
1233   }
1234 
1235   /**
1236    * A {@link PathFilter} that only allows directories.
1237    */
1238   public static class DirFilter extends BlackListDirFilter {
1239 
1240     public DirFilter(FileSystem fs) {
1241       super(fs, null);
1242     }
1243   }
1244 
1245   /**
1246    * A {@link PathFilter} that returns usertable directories. To get all directories use the
1247    * {@link BlackListDirFilter} with a <tt>null</tt> blacklist
1248    */
1249   public static class UserTableDirFilter extends BlackListDirFilter {
1250 
1251     public UserTableDirFilter(FileSystem fs) {
1252       super(fs, HConstants.HBASE_NON_TABLE_DIRS);
1253     }
1254   }
1255 
1256   /**
1257    * Heuristic to determine whether is safe or not to open a file for append
1258    * Looks both for dfs.support.append and use reflection to search
1259    * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
1260    * @param conf
1261    * @return True if append support
1262    */
1263   public static boolean isAppendSupported(final Configuration conf) {
1264     boolean append = conf.getBoolean("dfs.support.append", false);
1265     if (append) {
1266       try {
1267         // TODO: The implementation that comes back when we do a createWriter
1268         // may not be using SequenceFile so the below is not a definitive test.
1269         // Will do for now (hdfs-200).
1270         SequenceFile.Writer.class.getMethod("syncFs", new Class<?> []{});
1271         append = true;
1272       } catch (SecurityException e) {
1273       } catch (NoSuchMethodException e) {
1274         append = false;
1275       }
1276     }
1277     if (!append) {
1278       // Look for the 0.21, 0.22, new-style append evidence.
1279       try {
1280         FSDataOutputStream.class.getMethod("hflush", new Class<?> []{});
1281         append = true;
1282       } catch (NoSuchMethodException e) {
1283         append = false;
1284       }
1285     }
1286     return append;
1287   }
1288 
1289   /**
1290    * @param conf
1291    * @return True if this filesystem whose scheme is 'hdfs'.
1292    * @throws IOException
1293    */
1294   public static boolean isHDFS(final Configuration conf) throws IOException {
1295     FileSystem fs = FileSystem.get(conf);
1296     String scheme = fs.getUri().getScheme();
1297     return scheme.equalsIgnoreCase("hdfs");
1298   }
1299 
1300   /**
1301    * Recover file lease. Used when a file might be suspect
1302    * to be had been left open by another process.
1303    * @param fs FileSystem handle
1304    * @param p Path of file to recover lease
1305    * @param conf Configuration handle
1306    * @throws IOException
1307    */
1308   public abstract void recoverFileLease(final FileSystem fs, final Path p,
1309       Configuration conf, CancelableProgressable reporter) throws IOException;
1310 
1311   public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
1312       throws IOException {
1313     List<Path> tableDirs = new LinkedList<Path>();
1314 
1315     for(FileStatus status :
1316         fs.globStatus(new Path(rootdir,
1317             new Path(HConstants.BASE_NAMESPACE_DIR, "*")))) {
1318       tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath()));
1319     }
1320     return tableDirs;
1321   }
1322 
1323   /**
1324    * @param fs
1325    * @param rootdir
1326    * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders such as
1327    * .logs, .oldlogs, .corrupt folders.
1328    * @throws IOException
1329    */
1330   public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)
1331       throws IOException {
1332     // presumes any directory under hbase.rootdir is a table
1333     FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs));
1334     List<Path> tabledirs = new ArrayList<Path>(dirs.length);
1335     for (FileStatus dir: dirs) {
1336       tabledirs.add(dir.getPath());
1337     }
1338     return tabledirs;
1339   }
1340 
1341   /**
1342    * Checks if the given path is the one with 'recovered.edits' dir.
1343    * @param path
1344    * @return True if recovered edits.
1345    */
1346   public static boolean isRecoveredEdits(Path path) {
1347     return path.toString().contains(HConstants.RECOVERED_EDITS_DIR);
1348   }
1349 
1350   /**
1351    * Filter for all dirs that don't start with '.'
1352    */
1353   public static class RegionDirFilter implements PathFilter {
1354     // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.
1355     final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
1356     final FileSystem fs;
1357 
1358     public RegionDirFilter(FileSystem fs) {
1359       this.fs = fs;
1360     }
1361 
1362     @Override
1363     public boolean accept(Path rd) {
1364       if (!regionDirPattern.matcher(rd.getName()).matches()) {
1365         return false;
1366       }
1367 
1368       try {
1369         return fs.getFileStatus(rd).isDir();
1370       } catch (IOException ioe) {
1371         // Maybe the file was moved or the fs was disconnected.
1372         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1373         return false;
1374       }
1375     }
1376   }
1377 
1378   /**
1379    * Given a particular table dir, return all the regiondirs inside it, excluding files such as
1380    * .tableinfo
1381    * @param fs A file system for the Path
1382    * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir>
1383    * @return List of paths to valid region directories in table dir.
1384    * @throws IOException
1385    */
1386   public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException {
1387     // assumes we are in a table dir.
1388     FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs));
1389     List<Path> regionDirs = new ArrayList<Path>(rds.length);
1390     for (FileStatus rdfs: rds) {
1391       Path rdPath = rdfs.getPath();
1392       regionDirs.add(rdPath);
1393     }
1394     return regionDirs;
1395   }
1396 
1397   /**
1398    * Filter for all dirs that are legal column family names.  This is generally used for colfam
1399    * dirs <hbase.rootdir>/<tabledir>/<regiondir>/<colfamdir>.
1400    */
1401   public static class FamilyDirFilter implements PathFilter {
1402     final FileSystem fs;
1403 
1404     public FamilyDirFilter(FileSystem fs) {
1405       this.fs = fs;
1406     }
1407 
1408     @Override
1409     public boolean accept(Path rd) {
1410       try {
1411         // throws IAE if invalid
1412         HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
1413       } catch (IllegalArgumentException iae) {
1414         // path name is an invalid family name and thus is excluded.
1415         return false;
1416       }
1417 
1418       try {
1419         return fs.getFileStatus(rd).isDir();
1420       } catch (IOException ioe) {
1421         // Maybe the file was moved or the fs was disconnected.
1422         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1423         return false;
1424       }
1425     }
1426   }
1427 
1428   /**
1429    * Given a particular region dir, return all the familydirs inside it
1430    *
1431    * @param fs A file system for the Path
1432    * @param regionDir Path to a specific region directory
1433    * @return List of paths to valid family directories in region dir.
1434    * @throws IOException
1435    */
1436   public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
1437     // assumes we are in a region dir.
1438     FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
1439     List<Path> familyDirs = new ArrayList<Path>(fds.length);
1440     for (FileStatus fdfs: fds) {
1441       Path fdPath = fdfs.getPath();
1442       familyDirs.add(fdPath);
1443     }
1444     return familyDirs;
1445   }
1446 
1447   /**
1448    * Filter for HFiles that excludes reference files.
1449    */
1450   public static class HFileFilter implements PathFilter {
1451     // This pattern will accept 0.90+ style hex hfies files but reject reference files
1452     final public static Pattern hfilePattern = Pattern.compile("^([0-9a-f]+)$");
1453 
1454     final FileSystem fs;
1455 
1456     public HFileFilter(FileSystem fs) {
1457       this.fs = fs;
1458     }
1459 
1460     @Override
1461     public boolean accept(Path rd) {
1462       if (!hfilePattern.matcher(rd.getName()).matches()) {
1463         return false;
1464       }
1465 
1466       try {
1467         // only files
1468         return !fs.getFileStatus(rd).isDir();
1469       } catch (IOException ioe) {
1470         // Maybe the file was moved or the fs was disconnected.
1471         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1472         return false;
1473       }
1474     }
1475   }
1476 
1477   /**
1478    * @param conf
1479    * @return Returns the filesystem of the hbase rootdir.
1480    * @throws IOException
1481    */
1482   public static FileSystem getCurrentFileSystem(Configuration conf)
1483   throws IOException {
1484     return getRootDir(conf).getFileSystem(conf);
1485   }
1486 
1487 
1488   /**
1489    * Runs through the HBase rootdir/tablename and creates a reverse lookup map for
1490    * table StoreFile names to the full Path.
1491    * <br>
1492    * Example...<br>
1493    * Key = 3944417774205889744  <br>
1494    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1495    *
1496    * @param map map to add values.  If null, this method will create and populate one to return
1497    * @param fs  The file system to use.
1498    * @param hbaseRootDir  The root directory to scan.
1499    * @param tableName name of the table to scan.
1500    * @return Map keyed by StoreFile name with a value of the full Path.
1501    * @throws IOException When scanning the directory fails.
1502    */
1503   public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
1504   final FileSystem fs, final Path hbaseRootDir, TableName tableName)
1505   throws IOException {
1506     if (map == null) {
1507       map = new HashMap<String, Path>();
1508     }
1509 
1510     // only include the directory paths to tables
1511     Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
1512     // Inside a table, there are compaction.dir directories to skip.  Otherwise, all else
1513     // should be regions.
1514     PathFilter df = new BlackListDirFilter(fs, HConstants.HBASE_NON_TABLE_DIRS);
1515     FileStatus[] regionDirs = fs.listStatus(tableDir);
1516     for (FileStatus regionDir : regionDirs) {
1517       Path dd = regionDir.getPath();
1518       if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
1519         continue;
1520       }
1521       // else its a region name, now look in region for families
1522       FileStatus[] familyDirs = fs.listStatus(dd, df);
1523       for (FileStatus familyDir : familyDirs) {
1524         Path family = familyDir.getPath();
1525         // now in family, iterate over the StoreFiles and
1526         // put in map
1527         FileStatus[] familyStatus = fs.listStatus(family);
1528         for (FileStatus sfStatus : familyStatus) {
1529           Path sf = sfStatus.getPath();
1530           map.put( sf.getName(), sf);
1531         }
1532       }
1533     }
1534     return map;
1535   }
1536 
1537 
1538   /**
1539    * Runs through the HBase rootdir and creates a reverse lookup map for
1540    * table StoreFile names to the full Path.
1541    * <br>
1542    * Example...<br>
1543    * Key = 3944417774205889744  <br>
1544    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1545    *
1546    * @param fs  The file system to use.
1547    * @param hbaseRootDir  The root directory to scan.
1548    * @return Map keyed by StoreFile name with a value of the full Path.
1549    * @throws IOException When scanning the directory fails.
1550    */
1551   public static Map<String, Path> getTableStoreFilePathMap(
1552     final FileSystem fs, final Path hbaseRootDir)
1553   throws IOException {
1554     Map<String, Path> map = new HashMap<String, Path>();
1555 
1556     // if this method looks similar to 'getTableFragmentation' that is because
1557     // it was borrowed from it.
1558 
1559     // only include the directory paths to tables
1560     for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
1561       getTableStoreFilePathMap(map, fs, hbaseRootDir,
1562           FSUtils.getTableName(tableDir));
1563     }
1564     return map;
1565   }
1566 
1567   /**
1568    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
1569    * This accommodates differences between hadoop versions, where hadoop 1
1570    * does not throw a FileNotFoundException, and return an empty FileStatus[]
1571    * while Hadoop 2 will throw FileNotFoundException.
1572    *
1573    * @param fs file system
1574    * @param dir directory
1575    * @param filter path filter
1576    * @return null if dir is empty or doesn't exist, otherwise FileStatus array
1577    */
1578   public static FileStatus [] listStatus(final FileSystem fs,
1579       final Path dir, final PathFilter filter) throws IOException {
1580     FileStatus [] status = null;
1581     try {
1582       status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
1583     } catch (FileNotFoundException fnfe) {
1584       // if directory doesn't exist, return null
1585       if (LOG.isTraceEnabled()) {
1586         LOG.trace(dir + " doesn't exist");
1587       }
1588     }
1589     if (status == null || status.length < 1) return null;
1590     return status;
1591   }
1592 
1593   /**
1594    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
1595    * This would accommodates differences between hadoop versions
1596    *
1597    * @param fs file system
1598    * @param dir directory
1599    * @return null if dir is empty or doesn't exist, otherwise FileStatus array
1600    */
1601   public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException {
1602     return listStatus(fs, dir, null);
1603   }
1604 
1605   /**
1606    * Calls fs.delete() and returns the value returned by the fs.delete()
1607    *
1608    * @param fs
1609    * @param path
1610    * @param recursive
1611    * @return the value returned by the fs.delete()
1612    * @throws IOException
1613    */
1614   public static boolean delete(final FileSystem fs, final Path path, final boolean recursive)
1615       throws IOException {
1616     return fs.delete(path, recursive);
1617   }
1618 
1619   /**
1620    * Calls fs.exists(). Checks if the specified path exists
1621    *
1622    * @param fs
1623    * @param path
1624    * @return the value returned by fs.exists()
1625    * @throws IOException
1626    */
1627   public static boolean isExists(final FileSystem fs, final Path path) throws IOException {
1628     return fs.exists(path);
1629   }
1630 
1631   /**
1632    * Throw an exception if an action is not permitted by a user on a file.
1633    *
1634    * @param ugi
1635    *          the user
1636    * @param file
1637    *          the file
1638    * @param action
1639    *          the action
1640    */
1641   public static void checkAccess(UserGroupInformation ugi, FileStatus file,
1642       FsAction action) throws AccessControlException {
1643     if (ugi.getShortUserName().equals(file.getOwner())) {
1644       if (file.getPermission().getUserAction().implies(action)) {
1645         return;
1646       }
1647     } else if (contains(ugi.getGroupNames(), file.getGroup())) {
1648       if (file.getPermission().getGroupAction().implies(action)) {
1649         return;
1650       }
1651     } else if (file.getPermission().getOtherAction().implies(action)) {
1652       return;
1653     }
1654     throw new AccessControlException("Permission denied:" + " action=" + action
1655         + " path=" + file.getPath() + " user=" + ugi.getShortUserName());
1656   }
1657 
1658   private static boolean contains(String[] groups, String user) {
1659     for (String group : groups) {
1660       if (group.equals(user)) {
1661         return true;
1662       }
1663     }
1664     return false;
1665   }
1666 
1667   /**
1668    * Log the current state of the filesystem from a certain root directory
1669    * @param fs filesystem to investigate
1670    * @param root root file/directory to start logging from
1671    * @param LOG log to output information
1672    * @throws IOException if an unexpected exception occurs
1673    */
1674   public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
1675       throws IOException {
1676     LOG.debug("Current file system:");
1677     logFSTree(LOG, fs, root, "|-");
1678   }
1679 
1680   /**
1681    * Recursive helper to log the state of the FS
1682    *
1683    * @see #logFileSystemState(FileSystem, Path, Log)
1684    */
1685   private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix)
1686       throws IOException {
1687     FileStatus[] files = FSUtils.listStatus(fs, root, null);
1688     if (files == null) return;
1689 
1690     for (FileStatus file : files) {
1691       if (file.isDir()) {
1692         LOG.debug(prefix + file.getPath().getName() + "/");
1693         logFSTree(LOG, fs, file.getPath(), prefix + "---");
1694       } else {
1695         LOG.debug(prefix + file.getPath().getName());
1696       }
1697     }
1698   }
1699 
1700   public static boolean renameAndSetModifyTime(final FileSystem fs, final Path src, final Path dest)
1701       throws IOException {
1702     // set the modify time for TimeToLive Cleaner
1703     fs.setTimes(src, EnvironmentEdgeManager.currentTimeMillis(), -1);
1704     return fs.rename(src, dest);
1705   }
1706 
1707   /**
1708    * This function is to scan the root path of the file system to get the
1709    * degree of locality for each region on each of the servers having at least
1710    * one block of that region.
1711    * This is used by the tool {@link RegionPlacementMaintainer}
1712    *
1713    * @param conf
1714    *          the configuration to use
1715    * @return the mapping from region encoded name to a map of server names to
1716    *           locality fraction
1717    * @throws IOException
1718    *           in case of file system errors or interrupts
1719    */
1720   public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1721       final Configuration conf) throws IOException {
1722     return getRegionDegreeLocalityMappingFromFS(
1723         conf, null,
1724         conf.getInt(THREAD_POOLSIZE, DEFAULT_THREAD_POOLSIZE));
1725 
1726   }
1727 
1728   /**
1729    * This function is to scan the root path of the file system to get the
1730    * degree of locality for each region on each of the servers having at least
1731    * one block of that region.
1732    *
1733    * @param conf
1734    *          the configuration to use
1735    * @param desiredTable
1736    *          the table you wish to scan locality for
1737    * @param threadPoolSize
1738    *          the thread pool size to use
1739    * @return the mapping from region encoded name to a map of server names to
1740    *           locality fraction
1741    * @throws IOException
1742    *           in case of file system errors or interrupts
1743    */
1744   public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1745       final Configuration conf, final String desiredTable, int threadPoolSize)
1746       throws IOException {
1747     Map<String, Map<String, Float>> regionDegreeLocalityMapping =
1748         new ConcurrentHashMap<String, Map<String, Float>>();
1749     getRegionLocalityMappingFromFS(conf, desiredTable, threadPoolSize, null,
1750         regionDegreeLocalityMapping);
1751     return regionDegreeLocalityMapping;
1752   }
1753 
1754   /**
1755    * This function is to scan the root path of the file system to get either the
1756    * mapping between the region name and its best locality region server or the
1757    * degree of locality of each region on each of the servers having at least
1758    * one block of that region. The output map parameters are both optional.
1759    *
1760    * @param conf
1761    *          the configuration to use
1762    * @param desiredTable
1763    *          the table you wish to scan locality for
1764    * @param threadPoolSize
1765    *          the thread pool size to use
1766    * @param regionToBestLocalityRSMapping
1767    *          the map into which to put the best locality mapping or null
1768    * @param regionDegreeLocalityMapping
1769    *          the map into which to put the locality degree mapping or null,
1770    *          must be a thread-safe implementation
1771    * @throws IOException
1772    *           in case of file system errors or interrupts
1773    */
1774   private static void getRegionLocalityMappingFromFS(
1775       final Configuration conf, final String desiredTable,
1776       int threadPoolSize,
1777       Map<String, String> regionToBestLocalityRSMapping,
1778       Map<String, Map<String, Float>> regionDegreeLocalityMapping)
1779       throws IOException {
1780     FileSystem fs =  FileSystem.get(conf);
1781     Path rootPath = FSUtils.getRootDir(conf);
1782     long startTime = EnvironmentEdgeManager.currentTimeMillis();
1783     Path queryPath;
1784     // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/*
1785     if (null == desiredTable) {
1786       queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/");
1787     } else {
1788       queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/");
1789     }
1790 
1791     // reject all paths that are not appropriate
1792     PathFilter pathFilter = new PathFilter() {
1793       @Override
1794       public boolean accept(Path path) {
1795         // this is the region name; it may get some noise data
1796         if (null == path) {
1797           return false;
1798         }
1799 
1800         // no parent?
1801         Path parent = path.getParent();
1802         if (null == parent) {
1803           return false;
1804         }
1805 
1806         String regionName = path.getName();
1807         if (null == regionName) {
1808           return false;
1809         }
1810 
1811         if (!regionName.toLowerCase().matches("[0-9a-f]+")) {
1812           return false;
1813         }
1814         return true;
1815       }
1816     };
1817 
1818     FileStatus[] statusList = fs.globStatus(queryPath, pathFilter);
1819 
1820     if (null == statusList) {
1821       return;
1822     } else {
1823       LOG.debug("Query Path: " + queryPath + " ; # list of files: " +
1824           statusList.length);
1825     }
1826 
1827     // lower the number of threads in case we have very few expected regions
1828     threadPoolSize = Math.min(threadPoolSize, statusList.length);
1829 
1830     // run in multiple threads
1831     ThreadPoolExecutor tpe = new ThreadPoolExecutor(threadPoolSize,
1832         threadPoolSize, 60, TimeUnit.SECONDS,
1833         new ArrayBlockingQueue<Runnable>(statusList.length));
1834     try {
1835       // ignore all file status items that are not of interest
1836       for (FileStatus regionStatus : statusList) {
1837         if (null == regionStatus) {
1838           continue;
1839         }
1840 
1841         if (!regionStatus.isDir()) {
1842           continue;
1843         }
1844 
1845         Path regionPath = regionStatus.getPath();
1846         if (null == regionPath) {
1847           continue;
1848         }
1849 
1850         tpe.execute(new FSRegionScanner(fs, regionPath,
1851             regionToBestLocalityRSMapping, regionDegreeLocalityMapping));
1852       }
1853     } finally {
1854       tpe.shutdown();
1855       int threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
1856           60 * 1000);
1857       try {
1858         // here we wait until TPE terminates, which is either naturally or by
1859         // exceptions in the execution of the threads
1860         while (!tpe.awaitTermination(threadWakeFrequency,
1861             TimeUnit.MILLISECONDS)) {
1862           // printing out rough estimate, so as to not introduce
1863           // AtomicInteger
1864           LOG.info("Locality checking is underway: { Scanned Regions : "
1865               + tpe.getCompletedTaskCount() + "/"
1866               + tpe.getTaskCount() + " }");
1867         }
1868       } catch (InterruptedException e) {
1869         throw new IOException(e);
1870       }
1871     }
1872 
1873     long overhead = EnvironmentEdgeManager.currentTimeMillis() - startTime;
1874     String overheadMsg = "Scan DFS for locality info takes " + overhead + " ms";
1875 
1876     LOG.info(overheadMsg);
1877   }
1878 
1879   /**
1880    * Do our short circuit read setup.
1881    * Checks buffer size to use and whether to do checksumming in hbase or hdfs.
1882    * @param conf
1883    */
1884   public static void setupShortCircuitRead(final Configuration conf) {
1885     // Check that the user has not set the "dfs.client.read.shortcircuit.skip.checksum" property.
1886     boolean shortCircuitSkipChecksum =
1887       conf.getBoolean("dfs.client.read.shortcircuit.skip.checksum", false);
1888     boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
1889     if (shortCircuitSkipChecksum) {
1890       LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not " +
1891         "be set to true." + (useHBaseChecksum ? " HBase checksum doesn't require " +
1892         "it, see https://issues.apache.org/jira/browse/HBASE-6868." : ""));
1893       assert !shortCircuitSkipChecksum; //this will fail if assertions are on
1894     }
1895     checkShortCircuitReadBufferSize(conf);
1896   }
1897 
1898   /**
1899    * Check if short circuit read buffer size is set and if not, set it to hbase value.
1900    * @param conf
1901    */
1902   public static void checkShortCircuitReadBufferSize(final Configuration conf) {
1903     final int defaultSize = HConstants.DEFAULT_BLOCKSIZE * 2;
1904     final int notSet = -1;
1905     // DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_KEY is only defined in h2
1906     final String dfsKey = "dfs.client.read.shortcircuit.buffer.size";
1907     int size = conf.getInt(dfsKey, notSet);
1908     // If a size is set, return -- we will use it.
1909     if (size != notSet) return;
1910     // But short circuit buffer size is normally not set.  Put in place the hbase wanted size.
1911     int hbaseSize = conf.getInt("hbase." + dfsKey, defaultSize);
1912     conf.setIfUnset(dfsKey, Integer.toString(hbaseSize));
1913   }
1914 }