View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.DataInputStream;
23  import java.io.EOFException;
24  import java.io.FileNotFoundException;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.io.InterruptedIOException;
28  import java.lang.reflect.InvocationTargetException;
29  import java.lang.reflect.Method;
30  import java.net.InetSocketAddress;
31  import java.net.URI;
32  import java.net.URISyntaxException;
33  import java.util.ArrayList;
34  import java.util.Collections;
35  import java.util.HashMap;
36  import java.util.LinkedList;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.concurrent.ArrayBlockingQueue;
40  import java.util.concurrent.ConcurrentHashMap;
41  import java.util.concurrent.ThreadPoolExecutor;
42  import java.util.concurrent.TimeUnit;
43  import java.util.regex.Pattern;
44  
45  import org.apache.commons.logging.Log;
46  import org.apache.commons.logging.LogFactory;
47  import org.apache.hadoop.hbase.classification.InterfaceAudience;
48  import org.apache.hadoop.conf.Configuration;
49  import org.apache.hadoop.fs.BlockLocation;
50  import org.apache.hadoop.fs.FSDataInputStream;
51  import org.apache.hadoop.fs.FSDataOutputStream;
52  import org.apache.hadoop.fs.FileStatus;
53  import org.apache.hadoop.fs.FileSystem;
54  import org.apache.hadoop.fs.Path;
55  import org.apache.hadoop.fs.PathFilter;
56  import org.apache.hadoop.fs.permission.FsAction;
57  import org.apache.hadoop.fs.permission.FsPermission;
58  import org.apache.hadoop.hbase.ClusterId;
59  import org.apache.hadoop.hbase.HColumnDescriptor;
60  import org.apache.hadoop.hbase.HConstants;
61  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
62  import org.apache.hadoop.hbase.HRegionInfo;
63  import org.apache.hadoop.hbase.RemoteExceptionHandler;
64  import org.apache.hadoop.hbase.TableName;
65  import org.apache.hadoop.hbase.exceptions.DeserializationException;
66  import org.apache.hadoop.hbase.fs.HFileSystem;
67  import org.apache.hadoop.hbase.master.HMaster;
68  import org.apache.hadoop.hbase.master.RegionPlacementMaintainer;
69  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
70  import org.apache.hadoop.hbase.security.AccessDeniedException;
71  import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
72  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73  import org.apache.hadoop.hbase.protobuf.generated.FSProtos;
74  import org.apache.hadoop.hbase.regionserver.HRegion;
75  import org.apache.hadoop.hdfs.DistributedFileSystem;
76  import org.apache.hadoop.hdfs.protocol.FSConstants;
77  import org.apache.hadoop.io.IOUtils;
78  import org.apache.hadoop.io.SequenceFile;
79  import org.apache.hadoop.security.UserGroupInformation;
80  import org.apache.hadoop.util.Progressable;
81  import org.apache.hadoop.util.ReflectionUtils;
82  import org.apache.hadoop.util.StringUtils;
83  
84  import com.google.common.primitives.Ints;
85  
86  /**
87   * Utility methods for interacting with the underlying file system.
88   */
89  @InterfaceAudience.Private
90  public abstract class FSUtils {
91    private static final Log LOG = LogFactory.getLog(FSUtils.class);
92  
93    /** Full access permissions (starting point for a umask) */
94    public static final String FULL_RWX_PERMISSIONS = "777";
95    private static final String THREAD_POOLSIZE = "hbase.client.localityCheck.threadPoolSize";
96    private static final int DEFAULT_THREAD_POOLSIZE = 2;
97  
98    /** Set to true on Windows platforms */
99    public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");
100 
101   protected FSUtils() {
102     super();
103   }
104 
105   /**
106    * Compare of path component. Does not consider schema; i.e. if schemas different but <code>path
107    * <code> starts with <code>rootPath<code>, then the function returns true
108    * @param rootPath
109    * @param path
110    * @return True if <code>path</code> starts with <code>rootPath</code>
111    */
112   public static boolean isStartingWithPath(final Path rootPath, final String path) {
113     String uriRootPath = rootPath.toUri().getPath();
114     String tailUriPath = (new Path(path)).toUri().getPath();
115     return tailUriPath.startsWith(uriRootPath);
116   }
117 
118   /**
119    * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
120    * '/a/b/c' part. Does not consider schema; i.e. if schemas different but path or subpath matches,
121    * the two will equate.
122    * @param pathToSearch Path we will be trying to match.
123    * @param pathTail
124    * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
125    */
126   public static boolean isMatchingTail(final Path pathToSearch, String pathTail) {
127     return isMatchingTail(pathToSearch, new Path(pathTail));
128   }
129 
130   /**
131    * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the
132    * '/a/b/c' part. If you passed in 'hdfs://a/b/c and b/c, it would return true.  Does not consider
133    * schema; i.e. if schemas different but path or subpath matches, the two will equate.
134    * @param pathToSearch Path we will be trying to match.
135    * @param pathTail
136    * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code>
137    */
138   public static boolean isMatchingTail(final Path pathToSearch, final Path pathTail) {
139     if (pathToSearch.depth() != pathTail.depth()) return false;
140     Path tailPath = pathTail;
141     String tailName;
142     Path toSearch = pathToSearch;
143     String toSearchName;
144     boolean result = false;
145     do {
146       tailName = tailPath.getName();
147       if (tailName == null || tailName.length() <= 0) {
148         result = true;
149         break;
150       }
151       toSearchName = toSearch.getName();
152       if (toSearchName == null || toSearchName.length() <= 0) break;
153       // Move up a parent on each path for next go around.  Path doesn't let us go off the end.
154       tailPath = tailPath.getParent();
155       toSearch = toSearch.getParent();
156     } while(tailName.equals(toSearchName));
157     return result;
158   }
159 
160   public static FSUtils getInstance(FileSystem fs, Configuration conf) {
161     String scheme = fs.getUri().getScheme();
162     if (scheme == null) {
163       LOG.warn("Could not find scheme for uri " +
164           fs.getUri() + ", default to hdfs");
165       scheme = "hdfs";
166     }
167     Class<?> fsUtilsClass = conf.getClass("hbase.fsutil." +
168         scheme + ".impl", FSHDFSUtils.class); // Default to HDFS impl
169     FSUtils fsUtils = (FSUtils)ReflectionUtils.newInstance(fsUtilsClass, conf);
170     return fsUtils;
171   }
172 
173   /**
174    * Delete if exists.
175    * @param fs filesystem object
176    * @param dir directory to delete
177    * @return True if deleted <code>dir</code>
178    * @throws IOException e
179    */
180   public static boolean deleteDirectory(final FileSystem fs, final Path dir)
181   throws IOException {
182     return fs.exists(dir) && fs.delete(dir, true);
183   }
184 
185   /**
186    * Delete the region directory if exists.
187    * @param conf
188    * @param hri
189    * @return True if deleted the region directory.
190    * @throws IOException
191    */
192   public static boolean deleteRegionDir(final Configuration conf, final HRegionInfo hri)
193   throws IOException {
194     Path rootDir = getRootDir(conf);
195     FileSystem fs = rootDir.getFileSystem(conf);
196     return deleteDirectory(fs,
197       new Path(getTableDir(rootDir, hri.getTable()), hri.getEncodedName()));
198   }
199 
200   /**
201    * Return the number of bytes that large input files should be optimally
202    * be split into to minimize i/o time.
203    *
204    * use reflection to search for getDefaultBlockSize(Path f)
205    * if the method doesn't exist, fall back to using getDefaultBlockSize()
206    *
207    * @param fs filesystem object
208    * @return the default block size for the path's filesystem
209    * @throws IOException e
210    */
211   public static long getDefaultBlockSize(final FileSystem fs, final Path path) throws IOException {
212     Method m = null;
213     Class<? extends FileSystem> cls = fs.getClass();
214     try {
215       m = cls.getMethod("getDefaultBlockSize", new Class<?>[] { Path.class });
216     } catch (NoSuchMethodException e) {
217       LOG.info("FileSystem doesn't support getDefaultBlockSize");
218     } catch (SecurityException e) {
219       LOG.info("Doesn't have access to getDefaultBlockSize on FileSystems", e);
220       m = null; // could happen on setAccessible()
221     }
222     if (m == null) {
223       return fs.getDefaultBlockSize();
224     } else {
225       try {
226         Object ret = m.invoke(fs, path);
227         return ((Long)ret).longValue();
228       } catch (Exception e) {
229         throw new IOException(e);
230       }
231     }
232   }
233 
234   /*
235    * Get the default replication.
236    *
237    * use reflection to search for getDefaultReplication(Path f)
238    * if the method doesn't exist, fall back to using getDefaultReplication()
239    *
240    * @param fs filesystem object
241    * @param f path of file
242    * @return default replication for the path's filesystem
243    * @throws IOException e
244    */
245   public static short getDefaultReplication(final FileSystem fs, final Path path) throws IOException {
246     Method m = null;
247     Class<? extends FileSystem> cls = fs.getClass();
248     try {
249       m = cls.getMethod("getDefaultReplication", new Class<?>[] { Path.class });
250     } catch (NoSuchMethodException e) {
251       LOG.info("FileSystem doesn't support getDefaultReplication");
252     } catch (SecurityException e) {
253       LOG.info("Doesn't have access to getDefaultReplication on FileSystems", e);
254       m = null; // could happen on setAccessible()
255     }
256     if (m == null) {
257       return fs.getDefaultReplication();
258     } else {
259       try {
260         Object ret = m.invoke(fs, path);
261         return ((Number)ret).shortValue();
262       } catch (Exception e) {
263         throw new IOException(e);
264       }
265     }
266   }
267 
268   /**
269    * Returns the default buffer size to use during writes.
270    *
271    * The size of the buffer should probably be a multiple of hardware
272    * page size (4096 on Intel x86), and it determines how much data is
273    * buffered during read and write operations.
274    *
275    * @param fs filesystem object
276    * @return default buffer size to use during writes
277    */
278   public static int getDefaultBufferSize(final FileSystem fs) {
279     return fs.getConf().getInt("io.file.buffer.size", 4096);
280   }
281 
282   /**
283    * Create the specified file on the filesystem. By default, this will:
284    * <ol>
285    * <li>overwrite the file if it exists</li>
286    * <li>apply the umask in the configuration (if it is enabled)</li>
287    * <li>use the fs configured buffer size (or 4096 if not set)</li>
288    * <li>use the configured column family replication or default replication if
289    * {@link HColumnDescriptor#DEFAULT_DFS_REPLICATION}</li>
290    * <li>use the default block size</li>
291    * <li>not track progress</li>
292    * </ol>
293    * @param conf configurations
294    * @param fs {@link FileSystem} on which to write the file
295    * @param path {@link Path} to the file to write
296    * @param perm permissions
297    * @param favoredNodes
298    * @return output stream to the created file
299    * @throws IOException if the file cannot be created
300    */
301   public static FSDataOutputStream create(Configuration conf, FileSystem fs, Path path,
302       FsPermission perm, InetSocketAddress[] favoredNodes) throws IOException {
303     if (fs instanceof HFileSystem) {
304       FileSystem backingFs = ((HFileSystem)fs).getBackingFs();
305       if (backingFs instanceof DistributedFileSystem) {
306         // Try to use the favoredNodes version via reflection to allow backwards-
307         // compatibility.
308         short replication = Short.parseShort(conf.get(HColumnDescriptor.DFS_REPLICATION,
309           String.valueOf(HColumnDescriptor.DEFAULT_DFS_REPLICATION)));
310         try {
311           return (FSDataOutputStream) (DistributedFileSystem.class.getDeclaredMethod("create",
312             Path.class, FsPermission.class, boolean.class, int.class, short.class, long.class,
313             Progressable.class, InetSocketAddress[].class).invoke(backingFs, path, perm, true,
314             getDefaultBufferSize(backingFs),
315             replication > 0 ? replication : getDefaultReplication(backingFs, path),
316             getDefaultBlockSize(backingFs, path), null, favoredNodes));
317         } catch (InvocationTargetException ite) {
318           // Function was properly called, but threw it's own exception.
319           throw new IOException(ite.getCause());
320         } catch (NoSuchMethodException e) {
321           LOG.debug("DFS Client does not support most favored nodes create; using default create");
322           if (LOG.isTraceEnabled()) LOG.trace("Ignoring; use default create", e);
323         } catch (IllegalArgumentException e) {
324           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
325         } catch (SecurityException e) {
326           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
327         } catch (IllegalAccessException e) {
328           LOG.debug("Ignoring (most likely Reflection related exception) " + e);
329         }
330       }
331     }
332     return create(fs, path, perm, true);
333   }
334 
335   /**
336    * Create the specified file on the filesystem. By default, this will:
337    * <ol>
338    * <li>apply the umask in the configuration (if it is enabled)</li>
339    * <li>use the fs configured buffer size (or 4096 if not set)</li>
340    * <li>use the default replication</li>
341    * <li>use the default block size</li>
342    * <li>not track progress</li>
343    * </ol>
344    *
345    * @param fs {@link FileSystem} on which to write the file
346    * @param path {@link Path} to the file to write
347    * @param perm
348    * @param overwrite Whether or not the created file should be overwritten.
349    * @return output stream to the created file
350    * @throws IOException if the file cannot be created
351    */
352   public static FSDataOutputStream create(FileSystem fs, Path path,
353       FsPermission perm, boolean overwrite) throws IOException {
354     if (LOG.isTraceEnabled()) {
355       LOG.trace("Creating file=" + path + " with permission=" + perm + ", overwrite=" + overwrite);
356     }
357     return fs.create(path, perm, overwrite, getDefaultBufferSize(fs),
358         getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null);
359   }
360 
361   /**
362    * Get the file permissions specified in the configuration, if they are
363    * enabled.
364    *
365    * @param fs filesystem that the file will be created on.
366    * @param conf configuration to read for determining if permissions are
367    *          enabled and which to use
368    * @param permssionConfKey property key in the configuration to use when
369    *          finding the permission
370    * @return the permission to use when creating a new file on the fs. If
371    *         special permissions are not specified in the configuration, then
372    *         the default permissions on the the fs will be returned.
373    */
374   public static FsPermission getFilePermissions(final FileSystem fs,
375       final Configuration conf, final String permssionConfKey) {
376     boolean enablePermissions = conf.getBoolean(
377         HConstants.ENABLE_DATA_FILE_UMASK, false);
378 
379     if (enablePermissions) {
380       try {
381         FsPermission perm = new FsPermission(FULL_RWX_PERMISSIONS);
382         // make sure that we have a mask, if not, go default.
383         String mask = conf.get(permssionConfKey);
384         if (mask == null)
385           return getFileDefault();
386         // appy the umask
387         FsPermission umask = new FsPermission(mask);
388         return perm.applyUMask(umask);
389       } catch (IllegalArgumentException e) {
390         LOG.warn(
391             "Incorrect umask attempted to be created: "
392                 + conf.get(permssionConfKey)
393                 + ", using default file permissions.", e);
394         return getFileDefault();
395       }
396     }
397     return getFileDefault();
398   }
399 
400   /**
401    * Get the default permission for file.
402    * This is the same method as FsPermission.getFileDefault() in Hadoop 2.
403    * We provide the method here to support compatibility with Hadoop 1.
404    * See HBASE-11061.  Would be better to do this as Interface in hadoop-compat
405    * w/ hadoop1 and hadoop2 implementations but punting on this since small
406    * risk this will change in 0.96/0.98 timeframe (only committed to these
407    * branches).
408    */
409   public static FsPermission getFileDefault() {
410     return new FsPermission((short)00666);
411   }
412 
413   /**
414    * Checks to see if the specified file system is available
415    *
416    * @param fs filesystem
417    * @throws IOException e
418    */
419   public static void checkFileSystemAvailable(final FileSystem fs)
420   throws IOException {
421     if (!(fs instanceof DistributedFileSystem)) {
422       return;
423     }
424     IOException exception = null;
425     DistributedFileSystem dfs = (DistributedFileSystem) fs;
426     try {
427       if (dfs.exists(new Path("/"))) {
428         return;
429       }
430     } catch (IOException e) {
431       exception = RemoteExceptionHandler.checkIOException(e);
432     }
433     try {
434       fs.close();
435     } catch (Exception e) {
436       LOG.error("file system close failed: ", e);
437     }
438     IOException io = new IOException("File system is not available");
439     io.initCause(exception);
440     throw io;
441   }
442 
443   /**
444    * We use reflection because {@link DistributedFileSystem#setSafeMode(
445    * FSConstants.SafeModeAction action, boolean isChecked)} is not in hadoop 1.1
446    *
447    * @param dfs
448    * @return whether we're in safe mode
449    * @throws IOException
450    */
451   private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {
452     boolean inSafeMode = false;
453     try {
454       Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class<?> []{
455           org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class});
456       inSafeMode = (Boolean) m.invoke(dfs,
457         org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true);
458     } catch (Exception e) {
459       if (e instanceof IOException) throw (IOException) e;
460 
461       // Check whether dfs is on safemode.
462       inSafeMode = dfs.setSafeMode(
463         org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET);
464     }
465     return inSafeMode;
466   }
467 
468   /**
469    * Check whether dfs is in safemode.
470    * @param conf
471    * @throws IOException
472    */
473   public static void checkDfsSafeMode(final Configuration conf)
474   throws IOException {
475     boolean isInSafeMode = false;
476     FileSystem fs = FileSystem.get(conf);
477     if (fs instanceof DistributedFileSystem) {
478       DistributedFileSystem dfs = (DistributedFileSystem)fs;
479       isInSafeMode = isInSafeMode(dfs);
480     }
481     if (isInSafeMode) {
482       throw new IOException("File system is in safemode, it can't be written now");
483     }
484   }
485 
486   /**
487    * Verifies current version of file system
488    *
489    * @param fs filesystem object
490    * @param rootdir root hbase directory
491    * @return null if no version file exists, version string otherwise.
492    * @throws IOException e
493    * @throws org.apache.hadoop.hbase.exceptions.DeserializationException
494    */
495   public static String getVersion(FileSystem fs, Path rootdir)
496   throws IOException, DeserializationException {
497     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
498     FileStatus[] status = null;
499     try {
500       // hadoop 2.0 throws FNFE if directory does not exist.
501       // hadoop 1.0 returns null if directory does not exist.
502       status = fs.listStatus(versionFile);
503     } catch (FileNotFoundException fnfe) {
504       return null;
505     }
506     if (status == null || status.length == 0) return null;
507     String version = null;
508     byte [] content = new byte [(int)status[0].getLen()];
509     FSDataInputStream s = fs.open(versionFile);
510     try {
511       IOUtils.readFully(s, content, 0, content.length);
512       if (ProtobufUtil.isPBMagicPrefix(content)) {
513         version = parseVersionFrom(content);
514       } else {
515         // Presume it pre-pb format.
516         InputStream is = new ByteArrayInputStream(content);
517         DataInputStream dis = new DataInputStream(is);
518         try {
519           version = dis.readUTF();
520         } finally {
521           dis.close();
522         }
523       }
524     } catch (EOFException eof) {
525       LOG.warn("Version file was empty, odd, will try to set it.");
526     } finally {
527       s.close();
528     }
529     return version;
530   }
531 
532   /**
533    * Parse the content of the ${HBASE_ROOTDIR}/hbase.version file.
534    * @param bytes The byte content of the hbase.version file.
535    * @return The version found in the file as a String.
536    * @throws DeserializationException
537    */
538   static String parseVersionFrom(final byte [] bytes)
539   throws DeserializationException {
540     ProtobufUtil.expectPBMagicPrefix(bytes);
541     int pblen = ProtobufUtil.lengthOfPBMagic();
542     FSProtos.HBaseVersionFileContent.Builder builder =
543       FSProtos.HBaseVersionFileContent.newBuilder();
544     try {
545       ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
546       return builder.getVersion();
547     } catch (IOException e) {
548       // Convert
549       throw new DeserializationException(e);
550     }
551   }
552 
553   /**
554    * Create the content to write into the ${HBASE_ROOTDIR}/hbase.version file.
555    * @param version Version to persist
556    * @return Serialized protobuf with <code>version</code> content and a bit of pb magic for a prefix.
557    */
558   static byte [] toVersionByteArray(final String version) {
559     FSProtos.HBaseVersionFileContent.Builder builder =
560       FSProtos.HBaseVersionFileContent.newBuilder();
561     return ProtobufUtil.prependPBMagic(builder.setVersion(version).build().toByteArray());
562   }
563 
564   /**
565    * Verifies current version of file system
566    *
567    * @param fs file system
568    * @param rootdir root directory of HBase installation
569    * @param message if true, issues a message on System.out
570    *
571    * @throws IOException e
572    * @throws DeserializationException
573    */
574   public static void checkVersion(FileSystem fs, Path rootdir, boolean message)
575   throws IOException, DeserializationException {
576     checkVersion(fs, rootdir, message, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
577   }
578 
579   /**
580    * Verifies current version of file system
581    *
582    * @param fs file system
583    * @param rootdir root directory of HBase installation
584    * @param message if true, issues a message on System.out
585    * @param wait wait interval
586    * @param retries number of times to retry
587    *
588    * @throws IOException e
589    * @throws DeserializationException
590    */
591   public static void checkVersion(FileSystem fs, Path rootdir,
592       boolean message, int wait, int retries)
593   throws IOException, DeserializationException {
594     String version = getVersion(fs, rootdir);
595     if (version == null) {
596       if (!metaRegionExists(fs, rootdir)) {
597         // rootDir is empty (no version file and no root region)
598         // just create new version file (HBASE-1195)
599         setVersion(fs, rootdir, wait, retries);
600         return;
601       }
602     } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) return;
603 
604     // version is deprecated require migration
605     // Output on stdout so user sees it in terminal.
606     String msg = "HBase file layout needs to be upgraded."
607       + " You have version " + version
608       + " and I want version " + HConstants.FILE_SYSTEM_VERSION
609       + ". Consult http://hbase.apache.org/book.html for further information about upgrading HBase."
610       + " Is your hbase.rootdir valid? If so, you may need to run "
611       + "'hbase hbck -fixVersionFile'.";
612     if (message) {
613       System.out.println("WARNING! " + msg);
614     }
615     throw new FileSystemVersionException(msg);
616   }
617 
618   /**
619    * Sets version of file system
620    *
621    * @param fs filesystem object
622    * @param rootdir hbase root
623    * @throws IOException e
624    */
625   public static void setVersion(FileSystem fs, Path rootdir)
626   throws IOException {
627     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,
628       HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
629   }
630 
631   /**
632    * Sets version of file system
633    *
634    * @param fs filesystem object
635    * @param rootdir hbase root
636    * @param wait time to wait for retry
637    * @param retries number of times to retry before failing
638    * @throws IOException e
639    */
640   public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries)
641   throws IOException {
642     setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);
643   }
644 
645 
646   /**
647    * Sets version of file system
648    *
649    * @param fs filesystem object
650    * @param rootdir hbase root directory
651    * @param version version to set
652    * @param wait time to wait for retry
653    * @param retries number of times to retry before throwing an IOException
654    * @throws IOException e
655    */
656   public static void setVersion(FileSystem fs, Path rootdir, String version,
657       int wait, int retries) throws IOException {
658     Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
659     Path tempVersionFile = new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY + Path.SEPARATOR +
660       HConstants.VERSION_FILE_NAME);
661     while (true) {
662       try {
663         // Write the version to a temporary file
664         FSDataOutputStream s = fs.create(tempVersionFile);
665         try {
666           s.write(toVersionByteArray(version));
667           s.close();
668           s = null;
669           // Move the temp version file to its normal location. Returns false
670           // if the rename failed. Throw an IOE in that case.
671           if (!fs.rename(tempVersionFile, versionFile)) {
672             throw new IOException("Unable to move temp version file to " + versionFile);
673           }
674         } finally {
675           // Cleaning up the temporary if the rename failed would be trying
676           // too hard. We'll unconditionally create it again the next time
677           // through anyway, files are overwritten by default by create().
678 
679           // Attempt to close the stream on the way out if it is still open.
680           try {
681             if (s != null) s.close();
682           } catch (IOException ignore) { }
683         }
684         LOG.info("Created version file at " + rootdir.toString() + " with version=" + version);
685         return;
686       } catch (IOException e) {
687         if (retries > 0) {
688           LOG.debug("Unable to create version file at " + rootdir.toString() + ", retrying", e);
689           fs.delete(versionFile, false);
690           try {
691             if (wait > 0) {
692               Thread.sleep(wait);
693             }
694           } catch (InterruptedException ex) {
695             // ignore
696           }
697           retries--;
698         } else {
699           throw e;
700         }
701       }
702     }
703   }
704 
705   /**
706    * Checks that a cluster ID file exists in the HBase root directory
707    * @param fs the root directory FileSystem
708    * @param rootdir the HBase root directory in HDFS
709    * @param wait how long to wait between retries
710    * @return <code>true</code> if the file exists, otherwise <code>false</code>
711    * @throws IOException if checking the FileSystem fails
712    */
713   public static boolean checkClusterIdExists(FileSystem fs, Path rootdir,
714       int wait) throws IOException {
715     while (true) {
716       try {
717         Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
718         return fs.exists(filePath);
719       } catch (IOException ioe) {
720         if (wait > 0) {
721           LOG.warn("Unable to check cluster ID file in " + rootdir.toString() +
722               ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe));
723           try {
724             Thread.sleep(wait);
725           } catch (InterruptedException ie) {
726             throw (InterruptedIOException)new InterruptedIOException().initCause(ie);
727           }
728         } else {
729           throw ioe;
730         }
731       }
732     }
733   }
734 
735   /**
736    * Returns the value of the unique cluster ID stored for this HBase instance.
737    * @param fs the root directory FileSystem
738    * @param rootdir the path to the HBase root directory
739    * @return the unique cluster identifier
740    * @throws IOException if reading the cluster ID file fails
741    */
742   public static ClusterId getClusterId(FileSystem fs, Path rootdir)
743   throws IOException {
744     Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
745     ClusterId clusterId = null;
746     FileStatus status = fs.exists(idPath)? fs.getFileStatus(idPath):  null;
747     if (status != null) {
748       int len = Ints.checkedCast(status.getLen());
749       byte [] content = new byte[len];
750       FSDataInputStream in = fs.open(idPath);
751       try {
752         in.readFully(content);
753       } catch (EOFException eof) {
754         LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
755       } finally{
756         in.close();
757       }
758       try {
759         clusterId = ClusterId.parseFrom(content);
760       } catch (DeserializationException e) {
761         throw new IOException("content=" + Bytes.toString(content), e);
762       }
763       // If not pb'd, make it so.
764       if (!ProtobufUtil.isPBMagicPrefix(content)) {
765         String cid = null;
766         in = fs.open(idPath);
767         try {
768           cid = in.readUTF();
769           clusterId = new ClusterId(cid);
770         } catch (EOFException eof) {
771           LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
772         } finally {
773           in.close();
774         }
775         rewriteAsPb(fs, rootdir, idPath, clusterId);
776       }
777       return clusterId;
778     } else {
779       LOG.warn("Cluster ID file does not exist at " + idPath.toString());
780     }
781     return clusterId;
782   }
783 
784   /**
785    * @param cid
786    * @throws IOException
787    */
788   private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p,
789       final ClusterId cid)
790   throws IOException {
791     // Rewrite the file as pb.  Move aside the old one first, write new
792     // then delete the moved-aside file.
793     Path movedAsideName = new Path(p + "." + System.currentTimeMillis());
794     if (!fs.rename(p, movedAsideName)) throw new IOException("Failed rename of " + p);
795     setClusterId(fs, rootdir, cid, 100);
796     if (!fs.delete(movedAsideName, false)) {
797       throw new IOException("Failed delete of " + movedAsideName);
798     }
799     LOG.debug("Rewrote the hbase.id file as pb");
800   }
801 
802   /**
803    * Writes a new unique identifier for this cluster to the "hbase.id" file
804    * in the HBase root directory
805    * @param fs the root directory FileSystem
806    * @param rootdir the path to the HBase root directory
807    * @param clusterId the unique identifier to store
808    * @param wait how long (in milliseconds) to wait between retries
809    * @throws IOException if writing to the FileSystem fails and no wait value
810    */
811   public static void setClusterId(FileSystem fs, Path rootdir, ClusterId clusterId,
812       int wait) throws IOException {
813     while (true) {
814       try {
815         Path idFile = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
816         Path tempIdFile = new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY +
817           Path.SEPARATOR + HConstants.CLUSTER_ID_FILE_NAME);
818         // Write the id file to a temporary location
819         FSDataOutputStream s = fs.create(tempIdFile);
820         try {
821           s.write(clusterId.toByteArray());
822           s.close();
823           s = null;
824           // Move the temporary file to its normal location. Throw an IOE if
825           // the rename failed
826           if (!fs.rename(tempIdFile, idFile)) {
827             throw new IOException("Unable to move temp version file to " + idFile);
828           }
829         } finally {
830           // Attempt to close the stream if still open on the way out
831           try {
832             if (s != null) s.close();
833           } catch (IOException ignore) { }
834         }
835         if (LOG.isDebugEnabled()) {
836           LOG.debug("Created cluster ID file at " + idFile.toString() + " with ID: " + clusterId);
837         }
838         return;
839       } catch (IOException ioe) {
840         if (wait > 0) {
841           LOG.warn("Unable to create cluster ID file in " + rootdir.toString() +
842               ", retrying in " + wait + "msec: " + StringUtils.stringifyException(ioe));
843           try {
844             Thread.sleep(wait);
845           } catch (InterruptedException ie) {
846             Thread.currentThread().interrupt();
847             break;
848           }
849         } else {
850           throw ioe;
851         }
852       }
853     }
854   }
855 
856   /**
857    * Verifies root directory path is a valid URI with a scheme
858    *
859    * @param root root directory path
860    * @return Passed <code>root</code> argument.
861    * @throws IOException if not a valid URI with a scheme
862    */
863   public static Path validateRootPath(Path root) throws IOException {
864     try {
865       URI rootURI = new URI(root.toString());
866       String scheme = rootURI.getScheme();
867       if (scheme == null) {
868         throw new IOException("Root directory does not have a scheme");
869       }
870       return root;
871     } catch (URISyntaxException e) {
872       IOException io = new IOException("Root directory path is not a valid " +
873         "URI -- check your " + HConstants.HBASE_DIR + " configuration");
874       io.initCause(e);
875       throw io;
876     }
877   }
878 
879   /**
880    * Checks for the presence of the root path (using the provided conf object) in the given path. If
881    * it exists, this method removes it and returns the String representation of remaining relative path.
882    * @param path
883    * @param conf
884    * @return String representation of the remaining relative path
885    * @throws IOException
886    */
887   public static String removeRootPath(Path path, final Configuration conf) throws IOException {
888     Path root = FSUtils.getRootDir(conf);
889     String pathStr = path.toString();
890     // check that the path is absolute... it has the root path in it.
891     if (!pathStr.startsWith(root.toString())) return pathStr;
892     // if not, return as it is.
893     return pathStr.substring(root.toString().length() + 1);// remove the "/" too.
894   }
895 
896   /**
897    * If DFS, check safe mode and if so, wait until we clear it.
898    * @param conf configuration
899    * @param wait Sleep between retries
900    * @throws IOException e
901    */
902   public static void waitOnSafeMode(final Configuration conf,
903     final long wait)
904   throws IOException {
905     FileSystem fs = FileSystem.get(conf);
906     if (!(fs instanceof DistributedFileSystem)) return;
907     DistributedFileSystem dfs = (DistributedFileSystem)fs;
908     // Make sure dfs is not in safe mode
909     while (isInSafeMode(dfs)) {
910       LOG.info("Waiting for dfs to exit safe mode...");
911       try {
912         Thread.sleep(wait);
913       } catch (InterruptedException e) {
914         //continue
915       }
916     }
917   }
918 
919   /**
920    * Return the 'path' component of a Path.  In Hadoop, Path is an URI.  This
921    * method returns the 'path' component of a Path's URI: e.g. If a Path is
922    * <code>hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir</code>,
923    * this method returns <code>/hbase_trunk/TestTable/compaction.dir</code>.
924    * This method is useful if you want to print out a Path without qualifying
925    * Filesystem instance.
926    * @param p Filesystem Path whose 'path' component we are to return.
927    * @return Path portion of the Filesystem
928    */
929   public static String getPath(Path p) {
930     return p.toUri().getPath();
931   }
932 
933   /**
934    * @param c configuration
935    * @return Path to hbase root directory: i.e. <code>hbase.rootdir</code> from
936    * configuration as a qualified Path.
937    * @throws IOException e
938    */
939   public static Path getRootDir(final Configuration c) throws IOException {
940     Path p = new Path(c.get(HConstants.HBASE_DIR));
941     FileSystem fs = p.getFileSystem(c);
942     return p.makeQualified(fs);
943   }
944 
945   public static void setRootDir(final Configuration c, final Path root) throws IOException {
946     c.set(HConstants.HBASE_DIR, root.toString());
947   }
948 
949   public static void setFsDefault(final Configuration c, final Path root) throws IOException {
950     c.set("fs.defaultFS", root.toString());    // for hadoop 0.21+
951     c.set("fs.default.name", root.toString()); // for hadoop 0.20
952   }
953 
954   /**
955    * Checks if meta region exists
956    *
957    * @param fs file system
958    * @param rootdir root directory of HBase installation
959    * @return true if exists
960    * @throws IOException e
961    */
962   @SuppressWarnings("deprecation")
963   public static boolean metaRegionExists(FileSystem fs, Path rootdir)
964   throws IOException {
965     Path metaRegionDir =
966       HRegion.getRegionDir(rootdir, HRegionInfo.FIRST_META_REGIONINFO);
967     return fs.exists(metaRegionDir);
968   }
969 
970   /**
971    * Compute HDFS blocks distribution of a given file, or a portion of the file
972    * @param fs file system
973    * @param status file status of the file
974    * @param start start position of the portion
975    * @param length length of the portion
976    * @return The HDFS blocks distribution
977    */
978   static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
979     final FileSystem fs, FileStatus status, long start, long length)
980     throws IOException {
981     HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
982     BlockLocation [] blockLocations =
983       fs.getFileBlockLocations(status, start, length);
984     for(BlockLocation bl : blockLocations) {
985       String [] hosts = bl.getHosts();
986       long len = bl.getLength();
987       blocksDistribution.addHostsAndBlockWeight(hosts, len);
988     }
989 
990     return blocksDistribution;
991   }
992 
993 
994 
995   /**
996    * Runs through the hbase rootdir and checks all stores have only
997    * one file in them -- that is, they've been major compacted.  Looks
998    * at root and meta tables too.
999    * @param fs filesystem
1000    * @param hbaseRootDir hbase root directory
1001    * @return True if this hbase install is major compacted.
1002    * @throws IOException e
1003    */
1004   public static boolean isMajorCompacted(final FileSystem fs,
1005       final Path hbaseRootDir)
1006   throws IOException {
1007     List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
1008     PathFilter regionFilter = new RegionDirFilter(fs);
1009     PathFilter familyFilter = new FamilyDirFilter(fs);
1010     for (Path d : tableDirs) {
1011       FileStatus[] regionDirs = fs.listStatus(d, regionFilter);
1012       for (FileStatus regionDir : regionDirs) {
1013         Path dd = regionDir.getPath();
1014         // Else its a region name.  Now look in region for families.
1015         FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
1016         for (FileStatus familyDir : familyDirs) {
1017           Path family = familyDir.getPath();
1018           // Now in family make sure only one file.
1019           FileStatus[] familyStatus = fs.listStatus(family);
1020           if (familyStatus.length > 1) {
1021             LOG.debug(family.toString() + " has " + familyStatus.length +
1022                 " files.");
1023             return false;
1024           }
1025         }
1026       }
1027     }
1028     return true;
1029   }
1030 
1031   // TODO move this method OUT of FSUtils. No dependencies to HMaster
1032   /**
1033    * Returns the total overall fragmentation percentage. Includes hbase:meta and
1034    * -ROOT- as well.
1035    *
1036    * @param master  The master defining the HBase root and file system.
1037    * @return A map for each table and its percentage.
1038    * @throws IOException When scanning the directory fails.
1039    */
1040   public static int getTotalTableFragmentation(final HMaster master)
1041   throws IOException {
1042     Map<String, Integer> map = getTableFragmentation(master);
1043     return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
1044   }
1045 
1046   /**
1047    * Runs through the HBase rootdir and checks how many stores for each table
1048    * have more than one file in them. Checks -ROOT- and hbase:meta too. The total
1049    * percentage across all tables is stored under the special key "-TOTAL-".
1050    *
1051    * @param master  The master defining the HBase root and file system.
1052    * @return A map for each table and its percentage.
1053    *
1054    * @throws IOException When scanning the directory fails.
1055    */
1056   public static Map<String, Integer> getTableFragmentation(
1057     final HMaster master)
1058   throws IOException {
1059     Path path = getRootDir(master.getConfiguration());
1060     // since HMaster.getFileSystem() is package private
1061     FileSystem fs = path.getFileSystem(master.getConfiguration());
1062     return getTableFragmentation(fs, path);
1063   }
1064 
1065   /**
1066    * Runs through the HBase rootdir and checks how many stores for each table
1067    * have more than one file in them. Checks -ROOT- and hbase:meta too. The total
1068    * percentage across all tables is stored under the special key "-TOTAL-".
1069    *
1070    * @param fs  The file system to use.
1071    * @param hbaseRootDir  The root directory to scan.
1072    * @return A map for each table and its percentage.
1073    * @throws IOException When scanning the directory fails.
1074    */
1075   public static Map<String, Integer> getTableFragmentation(
1076     final FileSystem fs, final Path hbaseRootDir)
1077   throws IOException {
1078     Map<String, Integer> frags = new HashMap<String, Integer>();
1079     int cfCountTotal = 0;
1080     int cfFragTotal = 0;
1081     PathFilter regionFilter = new RegionDirFilter(fs);
1082     PathFilter familyFilter = new FamilyDirFilter(fs);
1083     List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
1084     for (Path d : tableDirs) {
1085       int cfCount = 0;
1086       int cfFrag = 0;
1087       FileStatus[] regionDirs = fs.listStatus(d, regionFilter);
1088       for (FileStatus regionDir : regionDirs) {
1089         Path dd = regionDir.getPath();
1090         // else its a region name, now look in region for families
1091         FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
1092         for (FileStatus familyDir : familyDirs) {
1093           cfCount++;
1094           cfCountTotal++;
1095           Path family = familyDir.getPath();
1096           // now in family make sure only one file
1097           FileStatus[] familyStatus = fs.listStatus(family);
1098           if (familyStatus.length > 1) {
1099             cfFrag++;
1100             cfFragTotal++;
1101           }
1102         }
1103       }
1104       // compute percentage per table and store in result list
1105       frags.put(FSUtils.getTableName(d).getNameAsString(),
1106           Math.round((float) cfFrag / cfCount * 100));
1107     }
1108     // set overall percentage for all tables
1109     frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100));
1110     return frags;
1111   }
1112 
1113   /**
1114    * Returns the {@link org.apache.hadoop.fs.Path} object representing the table directory under
1115    * path rootdir
1116    *
1117    * @param rootdir qualified path of HBase root directory
1118    * @param tableName name of table
1119    * @return {@link org.apache.hadoop.fs.Path} for table
1120    */
1121   public static Path getTableDir(Path rootdir, final TableName tableName) {
1122     return new Path(getNamespaceDir(rootdir, tableName.getNamespaceAsString()),
1123         tableName.getQualifierAsString());
1124   }
1125 
1126   /**
1127    * Returns the {@link org.apache.hadoop.hbase.TableName} object representing
1128    * the table directory under
1129    * path rootdir
1130    *
1131    * @param tablePath path of table
1132    * @return {@link org.apache.hadoop.fs.Path} for table
1133    */
1134   public static TableName getTableName(Path tablePath) {
1135     return TableName.valueOf(tablePath.getParent().getName(), tablePath.getName());
1136   }
1137 
1138   /**
1139    * Returns the {@link org.apache.hadoop.fs.Path} object representing
1140    * the namespace directory under path rootdir
1141    *
1142    * @param rootdir qualified path of HBase root directory
1143    * @param namespace namespace name
1144    * @return {@link org.apache.hadoop.fs.Path} for table
1145    */
1146   public static Path getNamespaceDir(Path rootdir, final String namespace) {
1147     return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR,
1148         new Path(namespace)));
1149   }
1150 
1151   /**
1152    * A {@link PathFilter} that returns only regular files.
1153    */
1154   static class FileFilter implements PathFilter {
1155     private final FileSystem fs;
1156 
1157     public FileFilter(final FileSystem fs) {
1158       this.fs = fs;
1159     }
1160 
1161     @Override
1162     public boolean accept(Path p) {
1163       try {
1164         return fs.isFile(p);
1165       } catch (IOException e) {
1166         LOG.debug("unable to verify if path=" + p + " is a regular file", e);
1167         return false;
1168       }
1169     }
1170   }
1171 
1172   /**
1173    * Directory filter that doesn't include any of the directories in the specified blacklist
1174    */
1175   public static class BlackListDirFilter implements PathFilter {
1176     private final FileSystem fs;
1177     private List<String> blacklist;
1178 
1179     /**
1180      * Create a filter on the give filesystem with the specified blacklist
1181      * @param fs filesystem to filter
1182      * @param directoryNameBlackList list of the names of the directories to filter. If
1183      *          <tt>null</tt>, all directories are returned
1184      */
1185     @SuppressWarnings("unchecked")
1186     public BlackListDirFilter(final FileSystem fs, final List<String> directoryNameBlackList) {
1187       this.fs = fs;
1188       blacklist =
1189         (List<String>) (directoryNameBlackList == null ? Collections.emptyList()
1190           : directoryNameBlackList);
1191     }
1192 
1193     @Override
1194     public boolean accept(Path p) {
1195       boolean isValid = false;
1196       try {
1197         if (isValidName(p.getName())) {
1198           isValid = fs.getFileStatus(p).isDir();
1199         } else {
1200           isValid = false;
1201         }
1202       } catch (IOException e) {
1203         LOG.warn("An error occurred while verifying if [" + p.toString()
1204             + "] is a valid directory. Returning 'not valid' and continuing.", e);
1205       }
1206       return isValid;
1207     }
1208 
1209     protected boolean isValidName(final String name) {
1210       return !blacklist.contains(name);
1211     }
1212   }
1213 
1214   /**
1215    * A {@link PathFilter} that only allows directories.
1216    */
1217   public static class DirFilter extends BlackListDirFilter {
1218 
1219     public DirFilter(FileSystem fs) {
1220       super(fs, null);
1221     }
1222   }
1223 
1224   /**
1225    * A {@link PathFilter} that returns usertable directories. To get all directories use the
1226    * {@link BlackListDirFilter} with a <tt>null</tt> blacklist
1227    */
1228   public static class UserTableDirFilter extends BlackListDirFilter {
1229     public UserTableDirFilter(FileSystem fs) {
1230       super(fs, HConstants.HBASE_NON_TABLE_DIRS);
1231     }
1232 
1233     protected boolean isValidName(final String name) {
1234       if (!super.isValidName(name))
1235         return false;
1236 
1237       try {
1238         TableName.isLegalTableQualifierName(Bytes.toBytes(name));
1239       } catch (IllegalArgumentException e) {
1240         LOG.info("INVALID NAME " + name);
1241         return false;
1242       }
1243       return true;
1244     }
1245   }
1246 
1247   /**
1248    * Heuristic to determine whether is safe or not to open a file for append
1249    * Looks both for dfs.support.append and use reflection to search
1250    * for SequenceFile.Writer.syncFs() or FSDataOutputStream.hflush()
1251    * @param conf
1252    * @return True if append support
1253    */
1254   public static boolean isAppendSupported(final Configuration conf) {
1255     boolean append = conf.getBoolean("dfs.support.append", false);
1256     if (append) {
1257       try {
1258         // TODO: The implementation that comes back when we do a createWriter
1259         // may not be using SequenceFile so the below is not a definitive test.
1260         // Will do for now (hdfs-200).
1261         SequenceFile.Writer.class.getMethod("syncFs", new Class<?> []{});
1262         append = true;
1263       } catch (SecurityException e) {
1264       } catch (NoSuchMethodException e) {
1265         append = false;
1266       }
1267     }
1268     if (!append) {
1269       // Look for the 0.21, 0.22, new-style append evidence.
1270       try {
1271         FSDataOutputStream.class.getMethod("hflush", new Class<?> []{});
1272         append = true;
1273       } catch (NoSuchMethodException e) {
1274         append = false;
1275       }
1276     }
1277     return append;
1278   }
1279 
1280   /**
1281    * @param conf
1282    * @return True if this filesystem whose scheme is 'hdfs'.
1283    * @throws IOException
1284    */
1285   public static boolean isHDFS(final Configuration conf) throws IOException {
1286     FileSystem fs = FileSystem.get(conf);
1287     String scheme = fs.getUri().getScheme();
1288     return scheme.equalsIgnoreCase("hdfs");
1289   }
1290 
1291   /**
1292    * Recover file lease. Used when a file might be suspect
1293    * to be had been left open by another process.
1294    * @param fs FileSystem handle
1295    * @param p Path of file to recover lease
1296    * @param conf Configuration handle
1297    * @throws IOException
1298    */
1299   public abstract void recoverFileLease(final FileSystem fs, final Path p,
1300       Configuration conf, CancelableProgressable reporter) throws IOException;
1301 
1302   public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
1303       throws IOException {
1304     List<Path> tableDirs = new LinkedList<Path>();
1305 
1306     for(FileStatus status :
1307         fs.globStatus(new Path(rootdir,
1308             new Path(HConstants.BASE_NAMESPACE_DIR, "*")))) {
1309       tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath()));
1310     }
1311     return tableDirs;
1312   }
1313 
1314   /**
1315    * @param fs
1316    * @param rootdir
1317    * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders such as
1318    * .logs, .oldlogs, .corrupt folders.
1319    * @throws IOException
1320    */
1321   public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)
1322       throws IOException {
1323     // presumes any directory under hbase.rootdir is a table
1324     FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs));
1325     List<Path> tabledirs = new ArrayList<Path>(dirs.length);
1326     for (FileStatus dir: dirs) {
1327       tabledirs.add(dir.getPath());
1328     }
1329     return tabledirs;
1330   }
1331 
1332   /**
1333    * Checks if the given path is the one with 'recovered.edits' dir.
1334    * @param path
1335    * @return True if we recovered edits
1336    */
1337   public static boolean isRecoveredEdits(Path path) {
1338     return path.toString().contains(HConstants.RECOVERED_EDITS_DIR);
1339   }
1340 
1341   /**
1342    * Filter for all dirs that don't start with '.'
1343    */
1344   public static class RegionDirFilter implements PathFilter {
1345     // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.
1346     final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
1347     final FileSystem fs;
1348 
1349     public RegionDirFilter(FileSystem fs) {
1350       this.fs = fs;
1351     }
1352 
1353     @Override
1354     public boolean accept(Path rd) {
1355       if (!regionDirPattern.matcher(rd.getName()).matches()) {
1356         return false;
1357       }
1358 
1359       try {
1360         return fs.getFileStatus(rd).isDir();
1361       } catch (IOException ioe) {
1362         // Maybe the file was moved or the fs was disconnected.
1363         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1364         return false;
1365       }
1366     }
1367   }
1368 
1369   /**
1370    * Given a particular table dir, return all the regiondirs inside it, excluding files such as
1371    * .tableinfo
1372    * @param fs A file system for the Path
1373    * @param tableDir Path to a specific table directory <hbase.rootdir>/<tabledir>
1374    * @return List of paths to valid region directories in table dir.
1375    * @throws IOException
1376    */
1377   public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException {
1378     // assumes we are in a table dir.
1379     FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs));
1380     List<Path> regionDirs = new ArrayList<Path>(rds.length);
1381     for (FileStatus rdfs: rds) {
1382       Path rdPath = rdfs.getPath();
1383       regionDirs.add(rdPath);
1384     }
1385     return regionDirs;
1386   }
1387 
1388   /**
1389    * Filter for all dirs that are legal column family names.  This is generally used for colfam
1390    * dirs <hbase.rootdir>/<tabledir>/<regiondir>/<colfamdir>.
1391    */
1392   public static class FamilyDirFilter implements PathFilter {
1393     final FileSystem fs;
1394 
1395     public FamilyDirFilter(FileSystem fs) {
1396       this.fs = fs;
1397     }
1398 
1399     @Override
1400     public boolean accept(Path rd) {
1401       try {
1402         // throws IAE if invalid
1403         HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
1404       } catch (IllegalArgumentException iae) {
1405         // path name is an invalid family name and thus is excluded.
1406         return false;
1407       }
1408 
1409       try {
1410         return fs.getFileStatus(rd).isDir();
1411       } catch (IOException ioe) {
1412         // Maybe the file was moved or the fs was disconnected.
1413         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1414         return false;
1415       }
1416     }
1417   }
1418 
1419   /**
1420    * Given a particular region dir, return all the familydirs inside it
1421    *
1422    * @param fs A file system for the Path
1423    * @param regionDir Path to a specific region directory
1424    * @return List of paths to valid family directories in region dir.
1425    * @throws IOException
1426    */
1427   public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
1428     // assumes we are in a region dir.
1429     FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
1430     List<Path> familyDirs = new ArrayList<Path>(fds.length);
1431     for (FileStatus fdfs: fds) {
1432       Path fdPath = fdfs.getPath();
1433       familyDirs.add(fdPath);
1434     }
1435     return familyDirs;
1436   }
1437 
1438   public static List<Path> getReferenceFilePaths(final FileSystem fs, final Path familyDir) throws IOException {
1439     FileStatus[] fds = fs.listStatus(familyDir, new ReferenceFileFilter(fs));
1440     List<Path> referenceFiles = new ArrayList<Path>(fds.length);
1441     for (FileStatus fdfs: fds) {
1442       Path fdPath = fdfs.getPath();
1443       referenceFiles.add(fdPath);
1444     }
1445     return referenceFiles;
1446   }
1447 
1448   /**
1449    * Filter for HFiles that excludes reference files.
1450    */
1451   public static class HFileFilter implements PathFilter {
1452     final FileSystem fs;
1453 
1454     public HFileFilter(FileSystem fs) {
1455       this.fs = fs;
1456     }
1457 
1458     @Override
1459     public boolean accept(Path rd) {
1460       try {
1461         // only files
1462         return !fs.getFileStatus(rd).isDir() && StoreFileInfo.isHFile(rd);
1463       } catch (IOException ioe) {
1464         // Maybe the file was moved or the fs was disconnected.
1465         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1466         return false;
1467       }
1468     }
1469   }
1470 
1471   public static class ReferenceFileFilter implements PathFilter {
1472 
1473     private final FileSystem fs;
1474 
1475     public ReferenceFileFilter(FileSystem fs) {
1476       this.fs = fs;
1477     }
1478 
1479     @Override
1480     public boolean accept(Path rd) {
1481       try {
1482         // only files can be references.
1483         return !fs.getFileStatus(rd).isDir() && StoreFileInfo.isReference(rd);
1484       } catch (IOException ioe) {
1485         // Maybe the file was moved or the fs was disconnected.
1486         LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1487         return false;
1488       }
1489     }
1490   }
1491 
1492 
1493   /**
1494    * @param conf
1495    * @return Returns the filesystem of the hbase rootdir.
1496    * @throws IOException
1497    */
1498   public static FileSystem getCurrentFileSystem(Configuration conf)
1499   throws IOException {
1500     return getRootDir(conf).getFileSystem(conf);
1501   }
1502 
1503 
1504   /**
1505    * Runs through the HBase rootdir/tablename and creates a reverse lookup map for
1506    * table StoreFile names to the full Path.
1507    * <br>
1508    * Example...<br>
1509    * Key = 3944417774205889744  <br>
1510    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1511    *
1512    * @param map map to add values.  If null, this method will create and populate one to return
1513    * @param fs  The file system to use.
1514    * @param hbaseRootDir  The root directory to scan.
1515    * @param tableName name of the table to scan.
1516    * @return Map keyed by StoreFile name with a value of the full Path.
1517    * @throws IOException When scanning the directory fails.
1518    */
1519   public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
1520   final FileSystem fs, final Path hbaseRootDir, TableName tableName)
1521   throws IOException {
1522     return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null);
1523   }
1524 
1525   /**
1526    * Runs through the HBase rootdir/tablename and creates a reverse lookup map for
1527    * table StoreFile names to the full Path.
1528    * <br>
1529    * Example...<br>
1530    * Key = 3944417774205889744  <br>
1531    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1532    *
1533    * @param map map to add values.  If null, this method will create and populate one to return
1534    * @param fs  The file system to use.
1535    * @param hbaseRootDir  The root directory to scan.
1536    * @param tableName name of the table to scan.
1537    * @param errors ErrorReporter instance or null
1538    * @return Map keyed by StoreFile name with a value of the full Path.
1539    * @throws IOException When scanning the directory fails.
1540    */
1541   public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
1542   final FileSystem fs, final Path hbaseRootDir, TableName tableName, ErrorReporter errors)
1543   throws IOException {
1544     if (map == null) {
1545       map = new HashMap<String, Path>();
1546     }
1547 
1548     // only include the directory paths to tables
1549     Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
1550     // Inside a table, there are compaction.dir directories to skip.  Otherwise, all else
1551     // should be regions.
1552     PathFilter familyFilter = new FamilyDirFilter(fs);
1553     FileStatus[] regionDirs = fs.listStatus(tableDir, new RegionDirFilter(fs));
1554     for (FileStatus regionDir : regionDirs) {
1555       if (null != errors) {
1556         errors.progress();
1557       }
1558       Path dd = regionDir.getPath();
1559       // else its a region name, now look in region for families
1560       FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);
1561       for (FileStatus familyDir : familyDirs) {
1562         if (null != errors) {
1563           errors.progress();
1564         }
1565         Path family = familyDir.getPath();
1566         // now in family, iterate over the StoreFiles and
1567         // put in map
1568         FileStatus[] familyStatus = fs.listStatus(family);
1569         for (FileStatus sfStatus : familyStatus) {
1570           if (null != errors) {
1571             errors.progress();
1572           }
1573           Path sf = sfStatus.getPath();
1574           map.put( sf.getName(), sf);
1575         }
1576       }
1577     }
1578     return map;
1579   }
1580 
1581   public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) {
1582     int result = 0;
1583     try {
1584       for (Path familyDir:getFamilyDirs(fs, p)){
1585         result += getReferenceFilePaths(fs, familyDir).size();
1586       }
1587     } catch (IOException e) {
1588       LOG.warn("Error Counting reference files.", e);
1589     }
1590     return result;
1591   }
1592 
1593   /**
1594    * Runs through the HBase rootdir and creates a reverse lookup map for
1595    * table StoreFile names to the full Path.
1596    * <br>
1597    * Example...<br>
1598    * Key = 3944417774205889744  <br>
1599    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1600    *
1601    * @param fs  The file system to use.
1602    * @param hbaseRootDir  The root directory to scan.
1603    * @return Map keyed by StoreFile name with a value of the full Path.
1604    * @throws IOException When scanning the directory fails.
1605    */
1606   public static Map<String, Path> getTableStoreFilePathMap(
1607     final FileSystem fs, final Path hbaseRootDir)
1608   throws IOException {
1609     return getTableStoreFilePathMap(fs, hbaseRootDir, null);
1610   }
1611 
1612   /**
1613    * Runs through the HBase rootdir and creates a reverse lookup map for
1614    * table StoreFile names to the full Path.
1615    * <br>
1616    * Example...<br>
1617    * Key = 3944417774205889744  <br>
1618    * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744
1619    *
1620    * @param fs  The file system to use.
1621    * @param hbaseRootDir  The root directory to scan.
1622    * @param errors ErrorReporter instance or null
1623    * @return Map keyed by StoreFile name with a value of the full Path.
1624    * @throws IOException When scanning the directory fails.
1625    */
1626   public static Map<String, Path> getTableStoreFilePathMap(
1627     final FileSystem fs, final Path hbaseRootDir, ErrorReporter errors)
1628   throws IOException {
1629     Map<String, Path> map = new HashMap<String, Path>();
1630 
1631     // if this method looks similar to 'getTableFragmentation' that is because
1632     // it was borrowed from it.
1633 
1634     // only include the directory paths to tables
1635     for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
1636       getTableStoreFilePathMap(map, fs, hbaseRootDir,
1637           FSUtils.getTableName(tableDir), errors);
1638     }
1639     return map;
1640   }
1641 
1642   /**
1643    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
1644    * This accommodates differences between hadoop versions, where hadoop 1
1645    * does not throw a FileNotFoundException, and return an empty FileStatus[]
1646    * while Hadoop 2 will throw FileNotFoundException.
1647    *
1648    * @param fs file system
1649    * @param dir directory
1650    * @param filter path filter
1651    * @return null if dir is empty or doesn't exist, otherwise FileStatus array
1652    */
1653   public static FileStatus [] listStatus(final FileSystem fs,
1654       final Path dir, final PathFilter filter) throws IOException {
1655     FileStatus [] status = null;
1656     try {
1657       status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
1658     } catch (FileNotFoundException fnfe) {
1659       // if directory doesn't exist, return null
1660       if (LOG.isTraceEnabled()) {
1661         LOG.trace(dir + " doesn't exist");
1662       }
1663     }
1664     if (status == null || status.length < 1) return null;
1665     return status;
1666   }
1667 
1668   /**
1669    * Calls fs.listStatus() and treats FileNotFoundException as non-fatal
1670    * This would accommodates differences between hadoop versions
1671    *
1672    * @param fs file system
1673    * @param dir directory
1674    * @return null if dir is empty or doesn't exist, otherwise FileStatus array
1675    */
1676   public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException {
1677     return listStatus(fs, dir, null);
1678   }
1679 
1680   /**
1681    * Calls fs.delete() and returns the value returned by the fs.delete()
1682    *
1683    * @param fs
1684    * @param path
1685    * @param recursive
1686    * @return the value returned by the fs.delete()
1687    * @throws IOException
1688    */
1689   public static boolean delete(final FileSystem fs, final Path path, final boolean recursive)
1690       throws IOException {
1691     return fs.delete(path, recursive);
1692   }
1693 
1694   /**
1695    * Calls fs.exists(). Checks if the specified path exists
1696    *
1697    * @param fs
1698    * @param path
1699    * @return the value returned by fs.exists()
1700    * @throws IOException
1701    */
1702   public static boolean isExists(final FileSystem fs, final Path path) throws IOException {
1703     return fs.exists(path);
1704   }
1705 
1706   /**
1707    * Throw an exception if an action is not permitted by a user on a file.
1708    *
1709    * @param ugi
1710    *          the user
1711    * @param file
1712    *          the file
1713    * @param action
1714    *          the action
1715    */
1716   public static void checkAccess(UserGroupInformation ugi, FileStatus file,
1717       FsAction action) throws AccessDeniedException {
1718     if (ugi.getShortUserName().equals(file.getOwner())) {
1719       if (file.getPermission().getUserAction().implies(action)) {
1720         return;
1721       }
1722     } else if (contains(ugi.getGroupNames(), file.getGroup())) {
1723       if (file.getPermission().getGroupAction().implies(action)) {
1724         return;
1725       }
1726     } else if (file.getPermission().getOtherAction().implies(action)) {
1727       return;
1728     }
1729     throw new AccessDeniedException("Permission denied:" + " action=" + action
1730         + " path=" + file.getPath() + " user=" + ugi.getShortUserName());
1731   }
1732 
1733   private static boolean contains(String[] groups, String user) {
1734     for (String group : groups) {
1735       if (group.equals(user)) {
1736         return true;
1737       }
1738     }
1739     return false;
1740   }
1741 
1742   /**
1743    * Log the current state of the filesystem from a certain root directory
1744    * @param fs filesystem to investigate
1745    * @param root root file/directory to start logging from
1746    * @param LOG log to output information
1747    * @throws IOException if an unexpected exception occurs
1748    */
1749   public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
1750       throws IOException {
1751     LOG.debug("Current file system:");
1752     logFSTree(LOG, fs, root, "|-");
1753   }
1754 
1755   /**
1756    * Recursive helper to log the state of the FS
1757    *
1758    * @see #logFileSystemState(FileSystem, Path, Log)
1759    */
1760   private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix)
1761       throws IOException {
1762     FileStatus[] files = FSUtils.listStatus(fs, root, null);
1763     if (files == null) return;
1764 
1765     for (FileStatus file : files) {
1766       if (file.isDir()) {
1767         LOG.debug(prefix + file.getPath().getName() + "/");
1768         logFSTree(LOG, fs, file.getPath(), prefix + "---");
1769       } else {
1770         LOG.debug(prefix + file.getPath().getName());
1771       }
1772     }
1773   }
1774 
1775   public static boolean renameAndSetModifyTime(final FileSystem fs, final Path src, final Path dest)
1776       throws IOException {
1777     // set the modify time for TimeToLive Cleaner
1778     fs.setTimes(src, EnvironmentEdgeManager.currentTimeMillis(), -1);
1779     return fs.rename(src, dest);
1780   }
1781 
1782   /**
1783    * This function is to scan the root path of the file system to get the
1784    * degree of locality for each region on each of the servers having at least
1785    * one block of that region.
1786    * This is used by the tool {@link RegionPlacementMaintainer}
1787    *
1788    * @param conf
1789    *          the configuration to use
1790    * @return the mapping from region encoded name to a map of server names to
1791    *           locality fraction
1792    * @throws IOException
1793    *           in case of file system errors or interrupts
1794    */
1795   public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1796       final Configuration conf) throws IOException {
1797     return getRegionDegreeLocalityMappingFromFS(
1798         conf, null,
1799         conf.getInt(THREAD_POOLSIZE, DEFAULT_THREAD_POOLSIZE));
1800 
1801   }
1802 
1803   /**
1804    * This function is to scan the root path of the file system to get the
1805    * degree of locality for each region on each of the servers having at least
1806    * one block of that region.
1807    *
1808    * @param conf
1809    *          the configuration to use
1810    * @param desiredTable
1811    *          the table you wish to scan locality for
1812    * @param threadPoolSize
1813    *          the thread pool size to use
1814    * @return the mapping from region encoded name to a map of server names to
1815    *           locality fraction
1816    * @throws IOException
1817    *           in case of file system errors or interrupts
1818    */
1819   public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1820       final Configuration conf, final String desiredTable, int threadPoolSize)
1821       throws IOException {
1822     Map<String, Map<String, Float>> regionDegreeLocalityMapping =
1823         new ConcurrentHashMap<String, Map<String, Float>>();
1824     getRegionLocalityMappingFromFS(conf, desiredTable, threadPoolSize, null,
1825         regionDegreeLocalityMapping);
1826     return regionDegreeLocalityMapping;
1827   }
1828 
1829   /**
1830    * This function is to scan the root path of the file system to get either the
1831    * mapping between the region name and its best locality region server or the
1832    * degree of locality of each region on each of the servers having at least
1833    * one block of that region. The output map parameters are both optional.
1834    *
1835    * @param conf
1836    *          the configuration to use
1837    * @param desiredTable
1838    *          the table you wish to scan locality for
1839    * @param threadPoolSize
1840    *          the thread pool size to use
1841    * @param regionToBestLocalityRSMapping
1842    *          the map into which to put the best locality mapping or null
1843    * @param regionDegreeLocalityMapping
1844    *          the map into which to put the locality degree mapping or null,
1845    *          must be a thread-safe implementation
1846    * @throws IOException
1847    *           in case of file system errors or interrupts
1848    */
1849   private static void getRegionLocalityMappingFromFS(
1850       final Configuration conf, final String desiredTable,
1851       int threadPoolSize,
1852       Map<String, String> regionToBestLocalityRSMapping,
1853       Map<String, Map<String, Float>> regionDegreeLocalityMapping)
1854       throws IOException {
1855     FileSystem fs =  FileSystem.get(conf);
1856     Path rootPath = FSUtils.getRootDir(conf);
1857     long startTime = EnvironmentEdgeManager.currentTimeMillis();
1858     Path queryPath;
1859     // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/*
1860     if (null == desiredTable) {
1861       queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/");
1862     } else {
1863       queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/");
1864     }
1865 
1866     // reject all paths that are not appropriate
1867     PathFilter pathFilter = new PathFilter() {
1868       @Override
1869       public boolean accept(Path path) {
1870         // this is the region name; it may get some noise data
1871         if (null == path) {
1872           return false;
1873         }
1874 
1875         // no parent?
1876         Path parent = path.getParent();
1877         if (null == parent) {
1878           return false;
1879         }
1880 
1881         String regionName = path.getName();
1882         if (null == regionName) {
1883           return false;
1884         }
1885 
1886         if (!regionName.toLowerCase().matches("[0-9a-f]+")) {
1887           return false;
1888         }
1889         return true;
1890       }
1891     };
1892 
1893     FileStatus[] statusList = fs.globStatus(queryPath, pathFilter);
1894 
1895     if (null == statusList) {
1896       return;
1897     } else {
1898       LOG.debug("Query Path: " + queryPath + " ; # list of files: " +
1899           statusList.length);
1900     }
1901 
1902     // lower the number of threads in case we have very few expected regions
1903     threadPoolSize = Math.min(threadPoolSize, statusList.length);
1904 
1905     // run in multiple threads
1906     ThreadPoolExecutor tpe = new ThreadPoolExecutor(threadPoolSize,
1907         threadPoolSize, 60, TimeUnit.SECONDS,
1908         new ArrayBlockingQueue<Runnable>(statusList.length));
1909     try {
1910       // ignore all file status items that are not of interest
1911       for (FileStatus regionStatus : statusList) {
1912         if (null == regionStatus) {
1913           continue;
1914         }
1915 
1916         if (!regionStatus.isDir()) {
1917           continue;
1918         }
1919 
1920         Path regionPath = regionStatus.getPath();
1921         if (null == regionPath) {
1922           continue;
1923         }
1924 
1925         tpe.execute(new FSRegionScanner(fs, regionPath,
1926             regionToBestLocalityRSMapping, regionDegreeLocalityMapping));
1927       }
1928     } finally {
1929       tpe.shutdown();
1930       int threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,
1931           60 * 1000);
1932       try {
1933         // here we wait until TPE terminates, which is either naturally or by
1934         // exceptions in the execution of the threads
1935         while (!tpe.awaitTermination(threadWakeFrequency,
1936             TimeUnit.MILLISECONDS)) {
1937           // printing out rough estimate, so as to not introduce
1938           // AtomicInteger
1939           LOG.info("Locality checking is underway: { Scanned Regions : "
1940               + tpe.getCompletedTaskCount() + "/"
1941               + tpe.getTaskCount() + " }");
1942         }
1943       } catch (InterruptedException e) {
1944         throw (InterruptedIOException)new InterruptedIOException().initCause(e);
1945       }
1946     }
1947 
1948     long overhead = EnvironmentEdgeManager.currentTimeMillis() - startTime;
1949     String overheadMsg = "Scan DFS for locality info takes " + overhead + " ms";
1950 
1951     LOG.info(overheadMsg);
1952   }
1953 
1954   /**
1955    * Do our short circuit read setup.
1956    * Checks buffer size to use and whether to do checksumming in hbase or hdfs.
1957    * @param conf
1958    */
1959   public static void setupShortCircuitRead(final Configuration conf) {
1960     // Check that the user has not set the "dfs.client.read.shortcircuit.skip.checksum" property.
1961     boolean shortCircuitSkipChecksum =
1962       conf.getBoolean("dfs.client.read.shortcircuit.skip.checksum", false);
1963     boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
1964     if (shortCircuitSkipChecksum) {
1965       LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not " +
1966         "be set to true." + (useHBaseChecksum ? " HBase checksum doesn't require " +
1967         "it, see https://issues.apache.org/jira/browse/HBASE-6868." : ""));
1968       assert !shortCircuitSkipChecksum; //this will fail if assertions are on
1969     }
1970     checkShortCircuitReadBufferSize(conf);
1971   }
1972 
1973   /**
1974    * Check if short circuit read buffer size is set and if not, set it to hbase value.
1975    * @param conf
1976    */
1977   public static void checkShortCircuitReadBufferSize(final Configuration conf) {
1978     final int defaultSize = HConstants.DEFAULT_BLOCKSIZE * 2;
1979     final int notSet = -1;
1980     // DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_KEY is only defined in h2
1981     final String dfsKey = "dfs.client.read.shortcircuit.buffer.size";
1982     int size = conf.getInt(dfsKey, notSet);
1983     // If a size is set, return -- we will use it.
1984     if (size != notSet) return;
1985     // But short circuit buffer size is normally not set.  Put in place the hbase wanted size.
1986     int hbaseSize = conf.getInt("hbase." + dfsKey, defaultSize);
1987     conf.setIfUnset(dfsKey, Integer.toString(hbaseSize));
1988   }
1989 }