1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.ByteArrayInputStream;
22 import java.io.DataInputStream;
23 import java.io.EOFException;
24 import java.io.FileNotFoundException;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.lang.reflect.InvocationTargetException;
28 import java.lang.reflect.Method;
29 import java.net.InetSocketAddress;
30 import java.net.URI;
31 import java.net.URISyntaxException;
32 import java.util.ArrayList;
33 import java.util.Collections;
34 import java.util.HashMap;
35 import java.util.LinkedList;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.concurrent.ArrayBlockingQueue;
39 import java.util.concurrent.ConcurrentHashMap;
40 import java.util.concurrent.ThreadPoolExecutor;
41 import java.util.concurrent.TimeUnit;
42 import java.util.regex.Pattern;
43
44 import org.apache.commons.logging.Log;
45 import org.apache.commons.logging.LogFactory;
46 import org.apache.hadoop.classification.InterfaceAudience;
47 import org.apache.hadoop.conf.Configuration;
48 import org.apache.hadoop.fs.BlockLocation;
49 import org.apache.hadoop.fs.FSDataInputStream;
50 import org.apache.hadoop.fs.FSDataOutputStream;
51 import org.apache.hadoop.fs.FileStatus;
52 import org.apache.hadoop.fs.FileSystem;
53 import org.apache.hadoop.fs.Path;
54 import org.apache.hadoop.fs.PathFilter;
55 import org.apache.hadoop.fs.permission.FsAction;
56 import org.apache.hadoop.fs.permission.FsPermission;
57 import org.apache.hadoop.hbase.ClusterId;
58 import org.apache.hadoop.hbase.HColumnDescriptor;
59 import org.apache.hadoop.hbase.HConstants;
60 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
61 import org.apache.hadoop.hbase.HRegionInfo;
62 import org.apache.hadoop.hbase.RemoteExceptionHandler;
63 import org.apache.hadoop.hbase.TableName;
64 import org.apache.hadoop.hbase.exceptions.DeserializationException;
65 import org.apache.hadoop.hbase.fs.HFileSystem;
66 import org.apache.hadoop.hbase.master.HMaster;
67 import org.apache.hadoop.hbase.master.RegionPlacementMaintainer;
68 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
69 import org.apache.hadoop.hbase.protobuf.generated.FSProtos;
70 import org.apache.hadoop.hbase.regionserver.HRegion;
71 import org.apache.hadoop.hdfs.DistributedFileSystem;
72 import org.apache.hadoop.hdfs.protocol.FSConstants;
73 import org.apache.hadoop.io.IOUtils;
74 import org.apache.hadoop.io.SequenceFile;
75 import org.apache.hadoop.security.AccessControlException;
76 import org.apache.hadoop.security.UserGroupInformation;
77 import org.apache.hadoop.util.Progressable;
78 import org.apache.hadoop.util.ReflectionUtils;
79 import org.apache.hadoop.util.StringUtils;
80
81 import com.google.common.primitives.Ints;
82 import com.google.protobuf.InvalidProtocolBufferException;
83
84
85
86
87 @InterfaceAudience.Private
88 public abstract class FSUtils {
89 private static final Log LOG = LogFactory.getLog(FSUtils.class);
90
91
92 private static final String FULL_RWX_PERMISSIONS = "777";
93 private static final String THREAD_POOLSIZE = "hbase.client.localityCheck.threadPoolSize";
94 private static final int DEFAULT_THREAD_POOLSIZE = 2;
95
96
97 public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");
98
99 protected FSUtils() {
100 super();
101 }
102
103
104
105
106
107
108
109
110 public static boolean isStartingWithPath(final Path rootPath, final String path) {
111 String uriRootPath = rootPath.toUri().getPath();
112 String tailUriPath = (new Path(path)).toUri().getPath();
113 return tailUriPath.startsWith(uriRootPath);
114 }
115
116
117
118
119
120
121
122
123
124 public static boolean isMatchingTail(final Path pathToSearch, String pathTail) {
125 return isMatchingTail(pathToSearch, new Path(pathTail));
126 }
127
128
129
130
131
132
133
134
135
136 public static boolean isMatchingTail(final Path pathToSearch, final Path pathTail) {
137 if (pathToSearch.depth() != pathTail.depth()) return false;
138 Path tailPath = pathTail;
139 String tailName;
140 Path toSearch = pathToSearch;
141 String toSearchName;
142 boolean result = false;
143 do {
144 tailName = tailPath.getName();
145 if (tailName == null || tailName.length() <= 0) {
146 result = true;
147 break;
148 }
149 toSearchName = toSearch.getName();
150 if (toSearchName == null || toSearchName.length() <= 0) break;
151
152 tailPath = tailPath.getParent();
153 toSearch = toSearch.getParent();
154 } while(tailName.equals(toSearchName));
155 return result;
156 }
157
158 public static FSUtils getInstance(FileSystem fs, Configuration conf) {
159 String scheme = fs.getUri().getScheme();
160 if (scheme == null) {
161 LOG.warn("Could not find scheme for uri " +
162 fs.getUri() + ", default to hdfs");
163 scheme = "hdfs";
164 }
165 Class<?> fsUtilsClass = conf.getClass("hbase.fsutil." +
166 scheme + ".impl", FSHDFSUtils.class);
167 FSUtils fsUtils = (FSUtils)ReflectionUtils.newInstance(fsUtilsClass, conf);
168 return fsUtils;
169 }
170
171
172
173
174
175
176
177
178 public static boolean deleteDirectory(final FileSystem fs, final Path dir)
179 throws IOException {
180 return fs.exists(dir) && fs.delete(dir, true);
181 }
182
183
184
185
186
187
188
189
190
191
192
193
194 public static long getDefaultBlockSize(final FileSystem fs, final Path path) throws IOException {
195 Method m = null;
196 Class<? extends FileSystem> cls = fs.getClass();
197 try {
198 m = cls.getMethod("getDefaultBlockSize", new Class<?>[] { Path.class });
199 } catch (NoSuchMethodException e) {
200 LOG.info("FileSystem doesn't support getDefaultBlockSize");
201 } catch (SecurityException e) {
202 LOG.info("Doesn't have access to getDefaultBlockSize on FileSystems", e);
203 m = null;
204 }
205 if (m == null) {
206 return fs.getDefaultBlockSize();
207 } else {
208 try {
209 Object ret = m.invoke(fs, path);
210 return ((Long)ret).longValue();
211 } catch (Exception e) {
212 throw new IOException(e);
213 }
214 }
215 }
216
217
218
219
220
221
222
223
224
225
226
227
228 public static short getDefaultReplication(final FileSystem fs, final Path path) throws IOException {
229 Method m = null;
230 Class<? extends FileSystem> cls = fs.getClass();
231 try {
232 m = cls.getMethod("getDefaultReplication", new Class<?>[] { Path.class });
233 } catch (NoSuchMethodException e) {
234 LOG.info("FileSystem doesn't support getDefaultReplication");
235 } catch (SecurityException e) {
236 LOG.info("Doesn't have access to getDefaultReplication on FileSystems", e);
237 m = null;
238 }
239 if (m == null) {
240 return fs.getDefaultReplication();
241 } else {
242 try {
243 Object ret = m.invoke(fs, path);
244 return ((Number)ret).shortValue();
245 } catch (Exception e) {
246 throw new IOException(e);
247 }
248 }
249 }
250
251
252
253
254
255
256
257
258
259
260
261 public static int getDefaultBufferSize(final FileSystem fs) {
262 return fs.getConf().getInt("io.file.buffer.size", 4096);
263 }
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283 public static FSDataOutputStream create(FileSystem fs, Path path,
284 FsPermission perm, InetSocketAddress[] favoredNodes) throws IOException {
285 if (fs instanceof HFileSystem) {
286 FileSystem backingFs = ((HFileSystem)fs).getBackingFs();
287 if (backingFs instanceof DistributedFileSystem) {
288
289
290 try {
291 return (FSDataOutputStream) (DistributedFileSystem.class
292 .getDeclaredMethod("create", Path.class, FsPermission.class,
293 boolean.class, int.class, short.class, long.class,
294 Progressable.class, InetSocketAddress[].class)
295 .invoke(backingFs, path, FsPermission.getDefault(), true,
296 getDefaultBufferSize(backingFs),
297 getDefaultReplication(backingFs, path),
298 getDefaultBlockSize(backingFs, path),
299 null, favoredNodes));
300 } catch (InvocationTargetException ite) {
301
302 throw new IOException(ite.getCause());
303 } catch (NoSuchMethodException e) {
304 LOG.debug("DFS Client does not support most favored nodes create; using default create");
305 if (LOG.isTraceEnabled()) LOG.trace("Ignoring; use default create", e);
306 } catch (IllegalArgumentException e) {
307 LOG.debug("Ignoring (most likely Reflection related exception) " + e);
308 } catch (SecurityException e) {
309 LOG.debug("Ignoring (most likely Reflection related exception) " + e);
310 } catch (IllegalAccessException e) {
311 LOG.debug("Ignoring (most likely Reflection related exception) " + e);
312 }
313 }
314 }
315 return create(fs, path, perm, true);
316 }
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335 public static FSDataOutputStream create(FileSystem fs, Path path,
336 FsPermission perm, boolean overwrite) throws IOException {
337 if (LOG.isTraceEnabled()) {
338 LOG.trace("Creating file=" + path + " with permission=" + perm + ", overwrite=" + overwrite);
339 }
340 return fs.create(path, perm, overwrite, getDefaultBufferSize(fs),
341 getDefaultReplication(fs, path), getDefaultBlockSize(fs, path), null);
342 }
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357 public static FsPermission getFilePermissions(final FileSystem fs,
358 final Configuration conf, final String permssionConfKey) {
359 boolean enablePermissions = conf.getBoolean(
360 HConstants.ENABLE_DATA_FILE_UMASK, false);
361
362 if (enablePermissions) {
363 try {
364 FsPermission perm = new FsPermission(FULL_RWX_PERMISSIONS);
365
366 String mask = conf.get(permssionConfKey);
367 if (mask == null)
368 return FsPermission.getDefault();
369
370 FsPermission umask = new FsPermission(mask);
371 return perm.applyUMask(umask);
372 } catch (IllegalArgumentException e) {
373 LOG.warn(
374 "Incorrect umask attempted to be created: "
375 + conf.get(permssionConfKey)
376 + ", using default file permissions.", e);
377 return FsPermission.getDefault();
378 }
379 }
380 return FsPermission.getDefault();
381 }
382
383
384
385
386
387
388
389 public static void checkFileSystemAvailable(final FileSystem fs)
390 throws IOException {
391 if (!(fs instanceof DistributedFileSystem)) {
392 return;
393 }
394 IOException exception = null;
395 DistributedFileSystem dfs = (DistributedFileSystem) fs;
396 try {
397 if (dfs.exists(new Path("/"))) {
398 return;
399 }
400 } catch (IOException e) {
401 exception = RemoteExceptionHandler.checkIOException(e);
402 }
403 try {
404 fs.close();
405 } catch (Exception e) {
406 LOG.error("file system close failed: ", e);
407 }
408 IOException io = new IOException("File system is not available");
409 io.initCause(exception);
410 throw io;
411 }
412
413
414
415
416
417
418
419
420
421 private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {
422 boolean inSafeMode = false;
423 try {
424 Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class<?> []{
425 org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class});
426 inSafeMode = (Boolean) m.invoke(dfs,
427 org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true);
428 } catch (Exception e) {
429 if (e instanceof IOException) throw (IOException) e;
430
431
432 inSafeMode = dfs.setSafeMode(
433 org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET);
434 }
435 return inSafeMode;
436 }
437
438
439
440
441
442
443 public static void checkDfsSafeMode(final Configuration conf)
444 throws IOException {
445 boolean isInSafeMode = false;
446 FileSystem fs = FileSystem.get(conf);
447 if (fs instanceof DistributedFileSystem) {
448 DistributedFileSystem dfs = (DistributedFileSystem)fs;
449 isInSafeMode = isInSafeMode(dfs);
450 }
451 if (isInSafeMode) {
452 throw new IOException("File system is in safemode, it can't be written now");
453 }
454 }
455
456
457
458
459
460
461
462
463
464
465 public static String getVersion(FileSystem fs, Path rootdir)
466 throws IOException, DeserializationException {
467 Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
468 FileStatus[] status = null;
469 try {
470
471
472 status = fs.listStatus(versionFile);
473 } catch (FileNotFoundException fnfe) {
474 return null;
475 }
476 if (status == null || status.length == 0) return null;
477 String version = null;
478 byte [] content = new byte [(int)status[0].getLen()];
479 FSDataInputStream s = fs.open(versionFile);
480 try {
481 IOUtils.readFully(s, content, 0, content.length);
482 if (ProtobufUtil.isPBMagicPrefix(content)) {
483 version = parseVersionFrom(content);
484 } else {
485
486 InputStream is = new ByteArrayInputStream(content);
487 DataInputStream dis = new DataInputStream(is);
488 try {
489 version = dis.readUTF();
490 } finally {
491 dis.close();
492 }
493
494 LOG.info("Updating the hbase.version file format with version=" + version);
495 setVersion(fs, rootdir, version, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
496 }
497 } catch (EOFException eof) {
498 LOG.warn("Version file was empty, odd, will try to set it.");
499 } finally {
500 s.close();
501 }
502 return version;
503 }
504
505
506
507
508
509
510
511 static String parseVersionFrom(final byte [] bytes)
512 throws DeserializationException {
513 ProtobufUtil.expectPBMagicPrefix(bytes);
514 int pblen = ProtobufUtil.lengthOfPBMagic();
515 FSProtos.HBaseVersionFileContent.Builder builder =
516 FSProtos.HBaseVersionFileContent.newBuilder();
517 FSProtos.HBaseVersionFileContent fileContent;
518 try {
519 fileContent = builder.mergeFrom(bytes, pblen, bytes.length - pblen).build();
520 return fileContent.getVersion();
521 } catch (InvalidProtocolBufferException e) {
522
523 throw new DeserializationException(e);
524 }
525 }
526
527
528
529
530
531
532 static byte [] toVersionByteArray(final String version) {
533 FSProtos.HBaseVersionFileContent.Builder builder =
534 FSProtos.HBaseVersionFileContent.newBuilder();
535 return ProtobufUtil.prependPBMagic(builder.setVersion(version).build().toByteArray());
536 }
537
538
539
540
541
542
543
544
545
546
547
548 public static void checkVersion(FileSystem fs, Path rootdir, boolean message)
549 throws IOException, DeserializationException {
550 checkVersion(fs, rootdir, message, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
551 }
552
553
554
555
556
557
558
559
560
561
562
563
564
565 public static void checkVersion(FileSystem fs, Path rootdir,
566 boolean message, int wait, int retries)
567 throws IOException, DeserializationException {
568 String version = getVersion(fs, rootdir);
569 if (version == null) {
570 if (!metaRegionExists(fs, rootdir)) {
571
572
573 setVersion(fs, rootdir, wait, retries);
574 return;
575 }
576 } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) return;
577
578
579
580 String msg = "HBase file layout needs to be upgraded."
581 + " You have version " + version
582 + " and I want version " + HConstants.FILE_SYSTEM_VERSION
583 + ". Is your hbase.rootdir valid? If so, you may need to run "
584 + "'hbase hbck -fixVersionFile'.";
585 if (message) {
586 System.out.println("WARNING! " + msg);
587 }
588 throw new FileSystemVersionException(msg);
589 }
590
591
592
593
594
595
596
597
598 public static void setVersion(FileSystem fs, Path rootdir)
599 throws IOException {
600 setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,
601 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
602 }
603
604
605
606
607
608
609
610
611
612
613 public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries)
614 throws IOException {
615 setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);
616 }
617
618
619
620
621
622
623
624
625
626
627
628
629 public static void setVersion(FileSystem fs, Path rootdir, String version,
630 int wait, int retries) throws IOException {
631 Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
632 while (true) {
633 try {
634 FSDataOutputStream s = fs.create(versionFile);
635 s.write(toVersionByteArray(version));
636 s.close();
637 LOG.debug("Created version file at " + rootdir.toString() + " with version=" + version);
638 return;
639 } catch (IOException e) {
640 if (retries > 0) {
641 LOG.warn("Unable to create version file at " + rootdir.toString() + ", retrying", e);
642 fs.delete(versionFile, false);
643 try {
644 if (wait > 0) {
645 Thread.sleep(wait);
646 }
647 } catch (InterruptedException ex) {
648
649 }
650 retries--;
651 } else {
652 throw e;
653 }
654 }
655 }
656 }
657
658
659
660
661
662
663
664
665
666 public static boolean checkClusterIdExists(FileSystem fs, Path rootdir,
667 int wait) throws IOException {
668 while (true) {
669 try {
670 Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
671 return fs.exists(filePath);
672 } catch (IOException ioe) {
673 if (wait > 0) {
674 LOG.warn("Unable to check cluster ID file in " + rootdir.toString() +
675 ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe));
676 try {
677 Thread.sleep(wait);
678 } catch (InterruptedException ie) {
679 Thread.interrupted();
680 break;
681 }
682 } else {
683 throw ioe;
684 }
685 }
686 }
687 return false;
688 }
689
690
691
692
693
694
695
696
697 public static ClusterId getClusterId(FileSystem fs, Path rootdir)
698 throws IOException {
699 Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
700 ClusterId clusterId = null;
701 FileStatus status = fs.exists(idPath)? fs.getFileStatus(idPath): null;
702 if (status != null) {
703 int len = Ints.checkedCast(status.getLen());
704 byte [] content = new byte[len];
705 FSDataInputStream in = fs.open(idPath);
706 try {
707 in.readFully(content);
708 } catch (EOFException eof) {
709 LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
710 } finally{
711 in.close();
712 }
713 try {
714 clusterId = ClusterId.parseFrom(content);
715 } catch (DeserializationException e) {
716 throw new IOException("content=" + Bytes.toString(content), e);
717 }
718
719 if (!ProtobufUtil.isPBMagicPrefix(content)) {
720 String cid = new String();
721 in = fs.open(idPath);
722 try {
723 cid = in.readUTF();
724 clusterId = new ClusterId(cid);
725 } catch (EOFException eof) {
726 LOG.warn("Cluster ID file " + idPath.toString() + " was empty");
727 } finally {
728 in.close();
729 }
730 rewriteAsPb(fs, rootdir, idPath, clusterId);
731 }
732 return clusterId;
733 } else {
734 LOG.warn("Cluster ID file does not exist at " + idPath.toString());
735 }
736 return clusterId;
737 }
738
739
740
741
742
743 private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p,
744 final ClusterId cid)
745 throws IOException {
746
747
748 Path movedAsideName = new Path(p + "." + System.currentTimeMillis());
749 if (!fs.rename(p, movedAsideName)) throw new IOException("Failed rename of " + p);
750 setClusterId(fs, rootdir, cid, 100);
751 if (!fs.delete(movedAsideName, false)) {
752 throw new IOException("Failed delete of " + movedAsideName);
753 }
754 LOG.debug("Rewrote the hbase.id file as pb");
755 }
756
757
758
759
760
761
762
763
764
765
766 public static void setClusterId(FileSystem fs, Path rootdir, ClusterId clusterId,
767 int wait) throws IOException {
768 while (true) {
769 try {
770 Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
771 FSDataOutputStream s = fs.create(filePath);
772 try {
773 s.write(clusterId.toByteArray());
774 } finally {
775 s.close();
776 }
777 if (LOG.isDebugEnabled()) {
778 LOG.debug("Created cluster ID file at " + filePath.toString() + " with ID: " + clusterId);
779 }
780 return;
781 } catch (IOException ioe) {
782 if (wait > 0) {
783 LOG.warn("Unable to create cluster ID file in " + rootdir.toString() +
784 ", retrying in " + wait + "msec: " + StringUtils.stringifyException(ioe));
785 try {
786 Thread.sleep(wait);
787 } catch (InterruptedException ie) {
788 Thread.interrupted();
789 break;
790 }
791 } else {
792 throw ioe;
793 }
794 }
795 }
796 }
797
798
799
800
801
802
803
804
805 public static Path validateRootPath(Path root) throws IOException {
806 try {
807 URI rootURI = new URI(root.toString());
808 String scheme = rootURI.getScheme();
809 if (scheme == null) {
810 throw new IOException("Root directory does not have a scheme");
811 }
812 return root;
813 } catch (URISyntaxException e) {
814 IOException io = new IOException("Root directory path is not a valid " +
815 "URI -- check your " + HConstants.HBASE_DIR + " configuration");
816 io.initCause(e);
817 throw io;
818 }
819 }
820
821
822
823
824
825
826
827
828
829 public static String removeRootPath(Path path, final Configuration conf) throws IOException {
830 Path root = FSUtils.getRootDir(conf);
831 String pathStr = path.toString();
832
833 if (!pathStr.startsWith(root.toString())) return pathStr;
834
835 return pathStr.substring(root.toString().length() + 1);
836 }
837
838
839
840
841
842
843
844 public static void waitOnSafeMode(final Configuration conf,
845 final long wait)
846 throws IOException {
847 FileSystem fs = FileSystem.get(conf);
848 if (!(fs instanceof DistributedFileSystem)) return;
849 DistributedFileSystem dfs = (DistributedFileSystem)fs;
850
851 while (isInSafeMode(dfs)) {
852 LOG.info("Waiting for dfs to exit safe mode...");
853 try {
854 Thread.sleep(wait);
855 } catch (InterruptedException e) {
856
857 }
858 }
859 }
860
861
862
863
864
865
866
867
868
869
870
871 public static String getPath(Path p) {
872 return p.toUri().getPath();
873 }
874
875
876
877
878
879
880
881 public static Path getRootDir(final Configuration c) throws IOException {
882 Path p = new Path(c.get(HConstants.HBASE_DIR));
883 FileSystem fs = p.getFileSystem(c);
884 return p.makeQualified(fs);
885 }
886
887 public static void setRootDir(final Configuration c, final Path root) throws IOException {
888 c.set(HConstants.HBASE_DIR, root.toString());
889 }
890
891 public static void setFsDefault(final Configuration c, final Path root) throws IOException {
892 c.set("fs.defaultFS", root.toString());
893 c.set("fs.default.name", root.toString());
894 }
895
896
897
898
899
900
901
902
903
904 @SuppressWarnings("deprecation")
905 public static boolean metaRegionExists(FileSystem fs, Path rootdir)
906 throws IOException {
907 Path metaRegionDir =
908 HRegion.getRegionDir(rootdir, HRegionInfo.FIRST_META_REGIONINFO);
909 return fs.exists(metaRegionDir);
910 }
911
912
913
914
915
916
917
918
919
920 static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
921 final FileSystem fs, FileStatus status, long start, long length)
922 throws IOException {
923 HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
924 BlockLocation [] blockLocations =
925 fs.getFileBlockLocations(status, start, length);
926 for(BlockLocation bl : blockLocations) {
927 String [] hosts = bl.getHosts();
928 long len = bl.getLength();
929 blocksDistribution.addHostsAndBlockWeight(hosts, len);
930 }
931
932 return blocksDistribution;
933 }
934
935
936
937
938
939
940
941
942
943
944
945
946 public static boolean isMajorCompacted(final FileSystem fs,
947 final Path hbaseRootDir)
948 throws IOException {
949 List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
950 for (Path d : tableDirs) {
951 FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
952 for (FileStatus regionDir : regionDirs) {
953 Path dd = regionDir.getPath();
954 if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
955 continue;
956 }
957
958 FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
959 for (FileStatus familyDir : familyDirs) {
960 Path family = familyDir.getPath();
961
962 FileStatus[] familyStatus = fs.listStatus(family);
963 if (familyStatus.length > 1) {
964 LOG.debug(family.toString() + " has " + familyStatus.length +
965 " files.");
966 return false;
967 }
968 }
969 }
970 }
971 return true;
972 }
973
974
975
976
977
978
979
980
981
982
983 public static int getTotalTableFragmentation(final HMaster master)
984 throws IOException {
985 Map<String, Integer> map = getTableFragmentation(master);
986 return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
987 }
988
989
990
991
992
993
994
995
996
997
998
999 public static Map<String, Integer> getTableFragmentation(
1000 final HMaster master)
1001 throws IOException {
1002 Path path = getRootDir(master.getConfiguration());
1003
1004 FileSystem fs = path.getFileSystem(master.getConfiguration());
1005 return getTableFragmentation(fs, path);
1006 }
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018 public static Map<String, Integer> getTableFragmentation(
1019 final FileSystem fs, final Path hbaseRootDir)
1020 throws IOException {
1021 Map<String, Integer> frags = new HashMap<String, Integer>();
1022 int cfCountTotal = 0;
1023 int cfFragTotal = 0;
1024 DirFilter df = new DirFilter(fs);
1025 List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
1026 for (Path d : tableDirs) {
1027 int cfCount = 0;
1028 int cfFrag = 0;
1029 FileStatus[] regionDirs = fs.listStatus(d, df);
1030 for (FileStatus regionDir : regionDirs) {
1031 Path dd = regionDir.getPath();
1032 if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
1033 continue;
1034 }
1035
1036 FileStatus[] familyDirs = fs.listStatus(dd, df);
1037 for (FileStatus familyDir : familyDirs) {
1038 cfCount++;
1039 cfCountTotal++;
1040 Path family = familyDir.getPath();
1041
1042 FileStatus[] familyStatus = fs.listStatus(family);
1043 if (familyStatus.length > 1) {
1044 cfFrag++;
1045 cfFragTotal++;
1046 }
1047 }
1048 }
1049
1050 frags.put(FSUtils.getTableName(d).getNameAsString(),
1051 Math.round((float) cfFrag / cfCount * 100));
1052 }
1053
1054 frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100));
1055 return frags;
1056 }
1057
1058
1059
1060
1061
1062
1063
1064
1065 public static boolean isPre020FileLayout(final FileSystem fs,
1066 final Path hbaseRootDir)
1067 throws IOException {
1068 Path mapfiles = new Path(new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
1069 "70236052"), "info"), "mapfiles");
1070 return fs.exists(mapfiles);
1071 }
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084 public static boolean isMajorCompactedPre020(final FileSystem fs,
1085 final Path hbaseRootDir)
1086 throws IOException {
1087
1088 List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);
1089 for (Path d: tableDirs) {
1090
1091
1092
1093
1094 if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
1095 continue;
1096 }
1097 FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
1098 for (FileStatus regionDir : regionDirs) {
1099 Path dd = regionDir.getPath();
1100 if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
1101 continue;
1102 }
1103
1104 FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
1105 for (FileStatus familyDir : familyDirs) {
1106 Path family = familyDir.getPath();
1107 FileStatus[] infoAndMapfile = fs.listStatus(family);
1108
1109 if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
1110 LOG.debug(family.toString() +
1111 " has more than just info and mapfile: " + infoAndMapfile.length);
1112 return false;
1113 }
1114
1115 for (int ll = 0; ll < 2; ll++) {
1116 if (infoAndMapfile[ll].getPath().getName().equals("info") ||
1117 infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
1118 continue;
1119 LOG.debug("Unexpected directory name: " +
1120 infoAndMapfile[ll].getPath());
1121 return false;
1122 }
1123
1124
1125 FileStatus[] familyStatus =
1126 fs.listStatus(new Path(family, "mapfiles"));
1127 if (familyStatus.length > 1) {
1128 LOG.debug(family.toString() + " has " + familyStatus.length +
1129 " files.");
1130 return false;
1131 }
1132 }
1133 }
1134 }
1135 return true;
1136 }
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146 public static Path getTableDir(Path rootdir, final TableName tableName) {
1147 return new Path(getNamespaceDir(rootdir, tableName.getNamespaceAsString()),
1148 tableName.getQualifierAsString());
1149 }
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159 public static TableName getTableName(Path tablePath) {
1160 return TableName.valueOf(tablePath.getParent().getName(), tablePath.getName());
1161 }
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171 public static Path getNamespaceDir(Path rootdir, final String namespace) {
1172 return new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR,
1173 new Path(namespace)));
1174 }
1175
1176
1177
1178
1179 static class FileFilter implements PathFilter {
1180 private final FileSystem fs;
1181
1182 public FileFilter(final FileSystem fs) {
1183 this.fs = fs;
1184 }
1185
1186 @Override
1187 public boolean accept(Path p) {
1188 try {
1189 return fs.isFile(p);
1190 } catch (IOException e) {
1191 LOG.debug("unable to verify if path=" + p + " is a regular file", e);
1192 return false;
1193 }
1194 }
1195 }
1196
1197
1198
1199
1200 public static class BlackListDirFilter implements PathFilter {
1201 private final FileSystem fs;
1202 private List<String> blacklist;
1203
1204
1205
1206
1207
1208
1209
1210 @SuppressWarnings("unchecked")
1211 public BlackListDirFilter(final FileSystem fs, final List<String> directoryNameBlackList) {
1212 this.fs = fs;
1213 blacklist =
1214 (List<String>) (directoryNameBlackList == null ? Collections.emptyList()
1215 : directoryNameBlackList);
1216 }
1217
1218 @Override
1219 public boolean accept(Path p) {
1220 boolean isValid = false;
1221 try {
1222 if (blacklist.contains(p.getName().toString())) {
1223 isValid = false;
1224 } else {
1225 isValid = fs.getFileStatus(p).isDir();
1226 }
1227 } catch (IOException e) {
1228 LOG.warn("An error occurred while verifying if [" + p.toString()
1229 + "] is a valid directory. Returning 'not valid' and continuing.", e);
1230 }
1231 return isValid;
1232 }
1233 }
1234
1235
1236
1237
1238 public static class DirFilter extends BlackListDirFilter {
1239
1240 public DirFilter(FileSystem fs) {
1241 super(fs, null);
1242 }
1243 }
1244
1245
1246
1247
1248
1249 public static class UserTableDirFilter extends BlackListDirFilter {
1250
1251 public UserTableDirFilter(FileSystem fs) {
1252 super(fs, HConstants.HBASE_NON_TABLE_DIRS);
1253 }
1254 }
1255
1256
1257
1258
1259
1260
1261
1262
1263 public static boolean isAppendSupported(final Configuration conf) {
1264 boolean append = conf.getBoolean("dfs.support.append", false);
1265 if (append) {
1266 try {
1267
1268
1269
1270 SequenceFile.Writer.class.getMethod("syncFs", new Class<?> []{});
1271 append = true;
1272 } catch (SecurityException e) {
1273 } catch (NoSuchMethodException e) {
1274 append = false;
1275 }
1276 }
1277 if (!append) {
1278
1279 try {
1280 FSDataOutputStream.class.getMethod("hflush", new Class<?> []{});
1281 append = true;
1282 } catch (NoSuchMethodException e) {
1283 append = false;
1284 }
1285 }
1286 return append;
1287 }
1288
1289
1290
1291
1292
1293
1294 public static boolean isHDFS(final Configuration conf) throws IOException {
1295 FileSystem fs = FileSystem.get(conf);
1296 String scheme = fs.getUri().getScheme();
1297 return scheme.equalsIgnoreCase("hdfs");
1298 }
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308 public abstract void recoverFileLease(final FileSystem fs, final Path p,
1309 Configuration conf, CancelableProgressable reporter) throws IOException;
1310
1311 public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
1312 throws IOException {
1313 List<Path> tableDirs = new LinkedList<Path>();
1314
1315 for(FileStatus status :
1316 fs.globStatus(new Path(rootdir,
1317 new Path(HConstants.BASE_NAMESPACE_DIR, "*")))) {
1318 tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath()));
1319 }
1320 return tableDirs;
1321 }
1322
1323
1324
1325
1326
1327
1328
1329
1330 public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)
1331 throws IOException {
1332
1333 FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs));
1334 List<Path> tabledirs = new ArrayList<Path>(dirs.length);
1335 for (FileStatus dir: dirs) {
1336 tabledirs.add(dir.getPath());
1337 }
1338 return tabledirs;
1339 }
1340
1341
1342
1343
1344
1345
1346 public static boolean isRecoveredEdits(Path path) {
1347 return path.toString().contains(HConstants.RECOVERED_EDITS_DIR);
1348 }
1349
1350
1351
1352
1353 public static class RegionDirFilter implements PathFilter {
1354
1355 final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
1356 final FileSystem fs;
1357
1358 public RegionDirFilter(FileSystem fs) {
1359 this.fs = fs;
1360 }
1361
1362 @Override
1363 public boolean accept(Path rd) {
1364 if (!regionDirPattern.matcher(rd.getName()).matches()) {
1365 return false;
1366 }
1367
1368 try {
1369 return fs.getFileStatus(rd).isDir();
1370 } catch (IOException ioe) {
1371
1372 LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1373 return false;
1374 }
1375 }
1376 }
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386 public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException {
1387
1388 FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs));
1389 List<Path> regionDirs = new ArrayList<Path>(rds.length);
1390 for (FileStatus rdfs: rds) {
1391 Path rdPath = rdfs.getPath();
1392 regionDirs.add(rdPath);
1393 }
1394 return regionDirs;
1395 }
1396
1397
1398
1399
1400
1401 public static class FamilyDirFilter implements PathFilter {
1402 final FileSystem fs;
1403
1404 public FamilyDirFilter(FileSystem fs) {
1405 this.fs = fs;
1406 }
1407
1408 @Override
1409 public boolean accept(Path rd) {
1410 try {
1411
1412 HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
1413 } catch (IllegalArgumentException iae) {
1414
1415 return false;
1416 }
1417
1418 try {
1419 return fs.getFileStatus(rd).isDir();
1420 } catch (IOException ioe) {
1421
1422 LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1423 return false;
1424 }
1425 }
1426 }
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436 public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
1437
1438 FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
1439 List<Path> familyDirs = new ArrayList<Path>(fds.length);
1440 for (FileStatus fdfs: fds) {
1441 Path fdPath = fdfs.getPath();
1442 familyDirs.add(fdPath);
1443 }
1444 return familyDirs;
1445 }
1446
1447
1448
1449
1450 public static class HFileFilter implements PathFilter {
1451
1452 final public static Pattern hfilePattern = Pattern.compile("^([0-9a-f]+)$");
1453
1454 final FileSystem fs;
1455
1456 public HFileFilter(FileSystem fs) {
1457 this.fs = fs;
1458 }
1459
1460 @Override
1461 public boolean accept(Path rd) {
1462 if (!hfilePattern.matcher(rd.getName()).matches()) {
1463 return false;
1464 }
1465
1466 try {
1467
1468 return !fs.getFileStatus(rd).isDir();
1469 } catch (IOException ioe) {
1470
1471 LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1472 return false;
1473 }
1474 }
1475 }
1476
1477
1478
1479
1480
1481
1482 public static FileSystem getCurrentFileSystem(Configuration conf)
1483 throws IOException {
1484 return getRootDir(conf).getFileSystem(conf);
1485 }
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503 public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,
1504 final FileSystem fs, final Path hbaseRootDir, TableName tableName)
1505 throws IOException {
1506 if (map == null) {
1507 map = new HashMap<String, Path>();
1508 }
1509
1510
1511 Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
1512
1513
1514 PathFilter df = new BlackListDirFilter(fs, HConstants.HBASE_NON_TABLE_DIRS);
1515 FileStatus[] regionDirs = fs.listStatus(tableDir);
1516 for (FileStatus regionDir : regionDirs) {
1517 Path dd = regionDir.getPath();
1518 if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
1519 continue;
1520 }
1521
1522 FileStatus[] familyDirs = fs.listStatus(dd, df);
1523 for (FileStatus familyDir : familyDirs) {
1524 Path family = familyDir.getPath();
1525
1526
1527 FileStatus[] familyStatus = fs.listStatus(family);
1528 for (FileStatus sfStatus : familyStatus) {
1529 Path sf = sfStatus.getPath();
1530 map.put( sf.getName(), sf);
1531 }
1532 }
1533 }
1534 return map;
1535 }
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551 public static Map<String, Path> getTableStoreFilePathMap(
1552 final FileSystem fs, final Path hbaseRootDir)
1553 throws IOException {
1554 Map<String, Path> map = new HashMap<String, Path>();
1555
1556
1557
1558
1559
1560 for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {
1561 getTableStoreFilePathMap(map, fs, hbaseRootDir,
1562 FSUtils.getTableName(tableDir));
1563 }
1564 return map;
1565 }
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578 public static FileStatus [] listStatus(final FileSystem fs,
1579 final Path dir, final PathFilter filter) throws IOException {
1580 FileStatus [] status = null;
1581 try {
1582 status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
1583 } catch (FileNotFoundException fnfe) {
1584
1585 if (LOG.isTraceEnabled()) {
1586 LOG.trace(dir + " doesn't exist");
1587 }
1588 }
1589 if (status == null || status.length < 1) return null;
1590 return status;
1591 }
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601 public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException {
1602 return listStatus(fs, dir, null);
1603 }
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614 public static boolean delete(final FileSystem fs, final Path path, final boolean recursive)
1615 throws IOException {
1616 return fs.delete(path, recursive);
1617 }
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627 public static boolean isExists(final FileSystem fs, final Path path) throws IOException {
1628 return fs.exists(path);
1629 }
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641 public static void checkAccess(UserGroupInformation ugi, FileStatus file,
1642 FsAction action) throws AccessControlException {
1643 if (ugi.getShortUserName().equals(file.getOwner())) {
1644 if (file.getPermission().getUserAction().implies(action)) {
1645 return;
1646 }
1647 } else if (contains(ugi.getGroupNames(), file.getGroup())) {
1648 if (file.getPermission().getGroupAction().implies(action)) {
1649 return;
1650 }
1651 } else if (file.getPermission().getOtherAction().implies(action)) {
1652 return;
1653 }
1654 throw new AccessControlException("Permission denied:" + " action=" + action
1655 + " path=" + file.getPath() + " user=" + ugi.getShortUserName());
1656 }
1657
1658 private static boolean contains(String[] groups, String user) {
1659 for (String group : groups) {
1660 if (group.equals(user)) {
1661 return true;
1662 }
1663 }
1664 return false;
1665 }
1666
1667
1668
1669
1670
1671
1672
1673
1674 public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
1675 throws IOException {
1676 LOG.debug("Current file system:");
1677 logFSTree(LOG, fs, root, "|-");
1678 }
1679
1680
1681
1682
1683
1684
1685 private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix)
1686 throws IOException {
1687 FileStatus[] files = FSUtils.listStatus(fs, root, null);
1688 if (files == null) return;
1689
1690 for (FileStatus file : files) {
1691 if (file.isDir()) {
1692 LOG.debug(prefix + file.getPath().getName() + "/");
1693 logFSTree(LOG, fs, file.getPath(), prefix + "---");
1694 } else {
1695 LOG.debug(prefix + file.getPath().getName());
1696 }
1697 }
1698 }
1699
1700 public static boolean renameAndSetModifyTime(final FileSystem fs, final Path src, final Path dest)
1701 throws IOException {
1702
1703 fs.setTimes(src, EnvironmentEdgeManager.currentTimeMillis(), -1);
1704 return fs.rename(src, dest);
1705 }
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720 public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1721 final Configuration conf) throws IOException {
1722 return getRegionDegreeLocalityMappingFromFS(
1723 conf, null,
1724 conf.getInt(THREAD_POOLSIZE, DEFAULT_THREAD_POOLSIZE));
1725
1726 }
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744 public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(
1745 final Configuration conf, final String desiredTable, int threadPoolSize)
1746 throws IOException {
1747 Map<String, Map<String, Float>> regionDegreeLocalityMapping =
1748 new ConcurrentHashMap<String, Map<String, Float>>();
1749 getRegionLocalityMappingFromFS(conf, desiredTable, threadPoolSize, null,
1750 regionDegreeLocalityMapping);
1751 return regionDegreeLocalityMapping;
1752 }
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774 private static void getRegionLocalityMappingFromFS(
1775 final Configuration conf, final String desiredTable,
1776 int threadPoolSize,
1777 Map<String, String> regionToBestLocalityRSMapping,
1778 Map<String, Map<String, Float>> regionDegreeLocalityMapping)
1779 throws IOException {
1780 FileSystem fs = FileSystem.get(conf);
1781 Path rootPath = FSUtils.getRootDir(conf);
1782 long startTime = EnvironmentEdgeManager.currentTimeMillis();
1783 Path queryPath;
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884 public static void setupShortCircuitRead(final Configuration conf) {
1885
1886 boolean shortCircuitSkipChecksum =
1887 conf.getBoolean("dfs.client.read.shortcircuit.skip.checksum", false);
1888 boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
1889 if (shortCircuitSkipChecksum) {
1890 LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not " +
1891 "be set to true." + (useHBaseChecksum ? " HBase checksum doesn't require " +
1892 "it, see https://issues.apache.org/jira/browse/HBASE-6868." : ""));
1893 assert !shortCircuitSkipChecksum;
1894 }
1895 checkShortCircuitReadBufferSize(conf);
1896 }
1897
1898
1899
1900
1901
1902 public static void checkShortCircuitReadBufferSize(final Configuration conf) {
1903 final int defaultSize = HConstants.DEFAULT_BLOCKSIZE * 2;
1904 final int notSet = -1;
1905
1906 final String dfsKey = "dfs.client.read.shortcircuit.buffer.size";
1907 int size = conf.getInt(dfsKey, notSet);
1908
1909 if (size != notSet) return;
1910
1911 int hbaseSize = conf.getInt("hbase." + dfsKey, defaultSize);
1912 conf.setIfUnset(dfsKey, Integer.toString(hbaseSize));
1913 }
1914 }