1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.util;
21
22 import java.io.DataInputStream;
23 import java.io.EOFException;
24 import java.io.FileNotFoundException;
25 import java.io.IOException;
26 import java.lang.reflect.Method;
27 import java.net.URI;
28 import java.net.URISyntaxException;
29 import java.util.ArrayList;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.regex.Pattern;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.BlockLocation;
39 import org.apache.hadoop.fs.FSDataInputStream;
40 import org.apache.hadoop.fs.FSDataOutputStream;
41 import org.apache.hadoop.fs.FileStatus;
42 import org.apache.hadoop.fs.FileSystem;
43 import org.apache.hadoop.fs.Path;
44 import org.apache.hadoop.fs.PathFilter;
45 import org.apache.hadoop.fs.permission.FsAction;
46 import org.apache.hadoop.fs.permission.FsPermission;
47 import org.apache.hadoop.hbase.HBaseFileSystem;
48 import org.apache.hadoop.hbase.HColumnDescriptor;
49 import org.apache.hadoop.hbase.HConstants;
50 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
51 import org.apache.hadoop.hbase.HRegionInfo;
52 import org.apache.hadoop.hbase.RemoteExceptionHandler;
53 import org.apache.hadoop.hbase.master.HMaster;
54 import org.apache.hadoop.hbase.regionserver.HRegion;
55 import org.apache.hadoop.hbase.security.User;
56 import org.apache.hadoop.hdfs.DistributedFileSystem;
57 import org.apache.hadoop.io.SequenceFile;
58 import org.apache.hadoop.security.AccessControlException;
59 import org.apache.hadoop.security.UserGroupInformation;
60 import org.apache.hadoop.util.ReflectionUtils;
61 import org.apache.hadoop.util.StringUtils;
62
63
64
65
66 public abstract class FSUtils {
67 private static final Log LOG = LogFactory.getLog(FSUtils.class);
68
69
70 private static final String FULL_RWX_PERMISSIONS = "777";
71
72 protected FSUtils() {
73 super();
74 }
75
76 public static FSUtils getInstance(FileSystem fs, Configuration conf) {
77 String scheme = fs.getUri().getScheme();
78 if (scheme == null) {
79 LOG.warn("Could not find scheme for uri " +
80 fs.getUri() + ", default to hdfs");
81 scheme = "hdfs";
82 }
83 Class<?> fsUtilsClass = conf.getClass("hbase.fsutil." +
84 scheme + ".impl", FSHDFSUtils.class);
85 FSUtils fsUtils = (FSUtils)ReflectionUtils.newInstance(fsUtilsClass, conf);
86 return fsUtils;
87 }
88
89
90
91
92
93
94
95
96 public static boolean deleteDirectory(final FileSystem fs, final Path dir)
97 throws IOException {
98 return fs.exists(dir) && fs.delete(dir, true);
99 }
100
101
102
103
104
105
106
107
108 public Path checkdir(final FileSystem fs, final Path dir) throws IOException {
109 if (!fs.exists(dir)) {
110 HBaseFileSystem.makeDirOnFileSystem(fs, dir);
111 }
112 return dir;
113 }
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132 public static FSDataOutputStream create(FileSystem fs, Path path,
133 FsPermission perm) throws IOException {
134 return create(fs, path, perm, true);
135 }
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155 public static FSDataOutputStream create(FileSystem fs, Path path, FsPermission perm,
156 boolean overwrite) throws IOException {
157 LOG.debug("Creating file=" + path + " with permission=" + perm);
158 return HBaseFileSystem.createPathWithPermsOnFileSystem(fs, path, perm, overwrite);
159 }
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174 public static FsPermission getFilePermissions(final FileSystem fs,
175 final Configuration conf, final String permssionConfKey) {
176 boolean enablePermissions = conf.getBoolean(
177 HConstants.ENABLE_DATA_FILE_UMASK, false);
178
179 if (enablePermissions) {
180 try {
181 FsPermission perm = new FsPermission(FULL_RWX_PERMISSIONS);
182
183 String mask = conf.get(permssionConfKey);
184 if (mask == null)
185 return FsPermission.getDefault();
186
187 FsPermission umask = new FsPermission(mask);
188 return perm.applyUMask(umask);
189 } catch (IllegalArgumentException e) {
190 LOG.warn(
191 "Incorrect umask attempted to be created: "
192 + conf.get(permssionConfKey)
193 + ", using default file permissions.", e);
194 return FsPermission.getDefault();
195 }
196 }
197 return FsPermission.getDefault();
198 }
199
200
201
202
203
204
205
206 public static void checkFileSystemAvailable(final FileSystem fs)
207 throws IOException {
208 if (!(fs instanceof DistributedFileSystem)) {
209 return;
210 }
211 IOException exception = null;
212 DistributedFileSystem dfs = (DistributedFileSystem) fs;
213 try {
214 if (dfs.exists(new Path("/"))) {
215 return;
216 }
217 } catch (IOException e) {
218 exception = RemoteExceptionHandler.checkIOException(e);
219 }
220 try {
221 fs.close();
222 } catch (Exception e) {
223 LOG.error("file system close failed: ", e);
224 }
225 IOException io = new IOException("File system is not available");
226 io.initCause(exception);
227 throw io;
228 }
229
230
231
232
233
234
235
236
237
238 private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {
239 boolean inSafeMode = false;
240 try {
241 Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class<?> []{
242 org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class});
243 inSafeMode = (Boolean) m.invoke(dfs,
244 org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true);
245 } catch (Exception e) {
246 if (e instanceof IOException) throw (IOException) e;
247
248
249 inSafeMode = dfs.setSafeMode(
250 org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET);
251 }
252 return inSafeMode;
253 }
254
255
256
257
258
259
260 public static void checkDfsSafeMode(final Configuration conf)
261 throws IOException {
262 boolean isInSafeMode = false;
263 FileSystem fs = FileSystem.get(conf);
264 if (fs instanceof DistributedFileSystem) {
265 DistributedFileSystem dfs = (DistributedFileSystem)fs;
266 isInSafeMode = isInSafeMode(dfs);
267 }
268 if (isInSafeMode) {
269 throw new IOException("File system is in safemode, it can't be written now");
270 }
271 }
272
273
274
275
276
277
278
279
280
281 public static String getVersion(FileSystem fs, Path rootdir)
282 throws IOException {
283 Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
284 String version = null;
285 if (fs.exists(versionFile)) {
286 FSDataInputStream s =
287 fs.open(versionFile);
288 try {
289 version = DataInputStream.readUTF(s);
290 } catch (EOFException eof) {
291 LOG.warn("Version file was empty, odd, will try to set it.");
292 } finally {
293 s.close();
294 }
295 }
296 return version;
297 }
298
299
300
301
302
303
304
305
306
307
308 public static void checkVersion(FileSystem fs, Path rootdir,
309 boolean message) throws IOException {
310 checkVersion(fs, rootdir, message, 0,
311 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
312 }
313
314
315
316
317
318
319
320
321
322
323
324
325 public static void checkVersion(FileSystem fs, Path rootdir,
326 boolean message, int wait, int retries) throws IOException {
327 String version = getVersion(fs, rootdir);
328
329 if (version == null) {
330 if (!rootRegionExists(fs, rootdir)) {
331
332
333 FSUtils.setVersion(fs, rootdir, wait, retries);
334 return;
335 }
336 } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0)
337 return;
338
339
340
341 String msg = "HBase file layout needs to be upgraded."
342 + " You have version " + version
343 + " and I want version " + HConstants.FILE_SYSTEM_VERSION
344 + ". Is your hbase.rootdir valid? If so, you may need to run "
345 + "'hbase hbck -fixVersionFile'.";
346 if (message) {
347 System.out.println("WARNING! " + msg);
348 }
349 throw new FileSystemVersionException(msg);
350 }
351
352
353
354
355
356
357
358
359 public static void setVersion(FileSystem fs, Path rootdir)
360 throws IOException {
361 setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,
362 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);
363 }
364
365
366
367
368
369
370
371
372
373
374 public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries)
375 throws IOException {
376 setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);
377 }
378
379
380
381
382
383
384
385
386
387
388
389
390 public static long getDefaultBlockSize(final FileSystem fs, final Path path) throws IOException {
391 Method m = null;
392 Class<? extends FileSystem> cls = fs.getClass();
393 try {
394 m = cls.getMethod("getDefaultBlockSize", new Class<?>[] { Path.class });
395 } catch (NoSuchMethodException e) {
396 LOG.info("FileSystem doesn't support getDefaultBlockSize");
397 } catch (SecurityException e) {
398 LOG.info("Doesn't have access to getDefaultBlockSize on FileSystems", e);
399 m = null;
400 }
401 if (m == null) {
402 return fs.getDefaultBlockSize();
403 } else {
404 try {
405 Object ret = m.invoke(fs, path);
406 return ((Long)ret).longValue();
407 } catch (Exception e) {
408 throw new IOException(e);
409 }
410 }
411 }
412
413
414
415
416
417
418
419
420
421
422
423
424 public static short getDefaultReplication(final FileSystem fs, final Path path) throws IOException {
425 Method m = null;
426 Class<? extends FileSystem> cls = fs.getClass();
427 try {
428 m = cls.getMethod("getDefaultReplication", new Class<?>[] { Path.class });
429 } catch (NoSuchMethodException e) {
430 LOG.info("FileSystem doesn't support getDefaultReplication");
431 } catch (SecurityException e) {
432 LOG.info("Doesn't have access to getDefaultReplication on FileSystems", e);
433 m = null;
434 }
435 if (m == null) {
436 return fs.getDefaultReplication();
437 } else {
438 try {
439 Object ret = m.invoke(fs, path);
440 return ((Number)ret).shortValue();
441 } catch (Exception e) {
442 throw new IOException(e);
443 }
444 }
445 }
446
447
448
449
450
451
452
453
454
455
456
457 public static int getDefaultBufferSize(final FileSystem fs) {
458 return fs.getConf().getInt("io.file.buffer.size", 4096);
459 }
460
461
462
463
464
465
466
467
468
469
470
471 public static void setVersion(FileSystem fs, Path rootdir, String version,
472 int wait, int retries) throws IOException {
473 Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
474 while (true) {
475 try {
476 FSDataOutputStream s = fs.create(versionFile);
477 s.writeUTF(version);
478 LOG.debug("Created version file at " + rootdir.toString() +
479 " set its version at:" + version);
480 s.close();
481 return;
482 } catch (IOException e) {
483 if (retries > 0) {
484 LOG.warn("Unable to create version file at " + rootdir.toString() +
485 ", retrying: " + e.getMessage());
486 fs.delete(versionFile, false);
487 try {
488 if (wait > 0) {
489 Thread.sleep(wait);
490 }
491 } catch (InterruptedException ex) {
492
493 }
494 retries--;
495 } else {
496 throw e;
497 }
498 }
499 }
500 }
501
502
503
504
505
506
507
508
509
510 public static boolean checkClusterIdExists(FileSystem fs, Path rootdir,
511 int wait) throws IOException {
512 while (true) {
513 try {
514 Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
515 return fs.exists(filePath);
516 } catch (IOException ioe) {
517 if (wait > 0) {
518 LOG.warn("Unable to check cluster ID file in " + rootdir.toString() +
519 ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe));
520 try {
521 Thread.sleep(wait);
522 } catch (InterruptedException ie) {
523 Thread.interrupted();
524 break;
525 }
526 } else {
527 throw ioe;
528 }
529 }
530 }
531 return false;
532 }
533
534
535
536
537
538
539
540
541 public static String getClusterId(FileSystem fs, Path rootdir)
542 throws IOException {
543 Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
544 String clusterId = null;
545 if (fs.exists(idPath)) {
546 FSDataInputStream in = fs.open(idPath);
547 try {
548 clusterId = in.readUTF();
549 } catch (EOFException eof) {
550 LOG.warn("Cluster ID file "+idPath.toString()+" was empty");
551 } finally{
552 in.close();
553 }
554 } else {
555 LOG.warn("Cluster ID file does not exist at " + idPath.toString());
556 }
557 return clusterId;
558 }
559
560
561
562
563
564
565
566
567
568
569 public static void setClusterId(FileSystem fs, Path rootdir, String clusterId,
570 int wait) throws IOException {
571 while (true) {
572 try {
573 Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);
574 FSDataOutputStream s = fs.create(filePath);
575 s.writeUTF(clusterId);
576 s.close();
577 if (LOG.isDebugEnabled()) {
578 LOG.debug("Created cluster ID file at " + filePath.toString() +
579 " with ID: " + clusterId);
580 }
581 return;
582 } catch (IOException ioe) {
583 if (wait > 0) {
584 LOG.warn("Unable to create cluster ID file in " + rootdir.toString() +
585 ", retrying in "+wait+"msec: "+StringUtils.stringifyException(ioe));
586 try {
587 Thread.sleep(wait);
588 } catch (InterruptedException ie) {
589 Thread.interrupted();
590 break;
591 }
592 } else {
593 throw ioe;
594 }
595 }
596 }
597 }
598
599
600
601
602
603
604
605
606 public static Path validateRootPath(Path root) throws IOException {
607 try {
608 URI rootURI = new URI(root.toString());
609 String scheme = rootURI.getScheme();
610 if (scheme == null) {
611 throw new IOException("Root directory does not have a scheme");
612 }
613 return root;
614 } catch (URISyntaxException e) {
615 IOException io = new IOException("Root directory path is not a valid " +
616 "URI -- check your " + HConstants.HBASE_DIR + " configuration");
617 io.initCause(e);
618 throw io;
619 }
620 }
621
622
623
624
625
626
627
628 public static void waitOnSafeMode(final Configuration conf,
629 final long wait)
630 throws IOException {
631 FileSystem fs = FileSystem.get(conf);
632 if (!(fs instanceof DistributedFileSystem)) return;
633 DistributedFileSystem dfs = (DistributedFileSystem)fs;
634
635 while (isInSafeMode(dfs)) {
636 LOG.info("Waiting for dfs to exit safe mode...");
637 try {
638 Thread.sleep(wait);
639 } catch (InterruptedException e) {
640
641 }
642 }
643 }
644
645
646
647
648
649
650
651
652
653
654
655 public static String getPath(Path p) {
656 return p.toUri().getPath();
657 }
658
659
660
661
662
663
664
665 public static Path getRootDir(final Configuration c) throws IOException {
666 Path p = new Path(c.get(HConstants.HBASE_DIR));
667 FileSystem fs = p.getFileSystem(c);
668 return p.makeQualified(fs);
669 }
670
671 public static void setRootDir(final Configuration c, final Path root) throws IOException {
672 c.set(HConstants.HBASE_DIR, root.toString());
673 }
674
675
676
677
678
679
680
681
682
683 public static boolean rootRegionExists(FileSystem fs, Path rootdir)
684 throws IOException {
685 Path rootRegionDir =
686 HRegion.getRegionDir(rootdir, HRegionInfo.ROOT_REGIONINFO);
687 return fs.exists(rootRegionDir);
688 }
689
690
691
692
693
694
695
696
697
698 static public HDFSBlocksDistribution computeHDFSBlocksDistribution(
699 final FileSystem fs, FileStatus status, long start, long length)
700 throws IOException {
701 HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();
702 BlockLocation [] blockLocations =
703 fs.getFileBlockLocations(status, start, length);
704 for(BlockLocation bl : blockLocations) {
705 String [] hosts = bl.getHosts();
706 long len = bl.getLength();
707 blocksDistribution.addHostsAndBlockWeight(hosts, len);
708 }
709
710 return blocksDistribution;
711 }
712
713
714
715
716
717
718
719
720
721
722
723
724 public static boolean isMajorCompacted(final FileSystem fs,
725 final Path hbaseRootDir)
726 throws IOException {
727
728 FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs));
729 for (FileStatus tableDir : tableDirs) {
730
731
732
733
734 Path d = tableDir.getPath();
735 if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
736 continue;
737 }
738 FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
739 for (FileStatus regionDir : regionDirs) {
740 Path dd = regionDir.getPath();
741 if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
742 continue;
743 }
744
745 FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
746 for (FileStatus familyDir : familyDirs) {
747 Path family = familyDir.getPath();
748
749 FileStatus[] familyStatus = fs.listStatus(family);
750 if (familyStatus.length > 1) {
751 LOG.debug(family.toString() + " has " + familyStatus.length +
752 " files.");
753 return false;
754 }
755 }
756 }
757 }
758 return true;
759 }
760
761
762
763
764
765
766
767
768
769
770 public static int getTotalTableFragmentation(final HMaster master)
771 throws IOException {
772 Map<String, Integer> map = getTableFragmentation(master);
773 return map != null && map.size() > 0 ? map.get("-TOTAL-") : -1;
774 }
775
776
777
778
779
780
781
782
783
784
785 public static Map<String, Integer> getTableFragmentation(
786 final HMaster master)
787 throws IOException {
788 Path path = getRootDir(master.getConfiguration());
789
790 FileSystem fs = path.getFileSystem(master.getConfiguration());
791 return getTableFragmentation(fs, path);
792 }
793
794
795
796
797
798
799
800
801
802
803
804 public static Map<String, Integer> getTableFragmentation(
805 final FileSystem fs, final Path hbaseRootDir)
806 throws IOException {
807 Map<String, Integer> frags = new HashMap<String, Integer>();
808 int cfCountTotal = 0;
809 int cfFragTotal = 0;
810 DirFilter df = new DirFilter(fs);
811
812 FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
813 for (FileStatus tableDir : tableDirs) {
814
815
816
817
818 Path d = tableDir.getPath();
819 if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
820 continue;
821 }
822 int cfCount = 0;
823 int cfFrag = 0;
824 FileStatus[] regionDirs = fs.listStatus(d, df);
825 for (FileStatus regionDir : regionDirs) {
826 Path dd = regionDir.getPath();
827 if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
828 continue;
829 }
830
831 FileStatus[] familyDirs = fs.listStatus(dd, df);
832 for (FileStatus familyDir : familyDirs) {
833 cfCount++;
834 cfCountTotal++;
835 Path family = familyDir.getPath();
836
837 FileStatus[] familyStatus = fs.listStatus(family);
838 if (familyStatus.length > 1) {
839 cfFrag++;
840 cfFragTotal++;
841 }
842 }
843 }
844
845 frags.put(d.getName(), Math.round((float) cfFrag / cfCount * 100));
846 }
847
848 frags.put("-TOTAL-", Math.round((float) cfFragTotal / cfCountTotal * 100));
849 return frags;
850 }
851
852
853
854
855
856
857
858
859 public static boolean isPre020FileLayout(final FileSystem fs,
860 final Path hbaseRootDir)
861 throws IOException {
862 Path mapfiles = new Path(new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
863 "70236052"), "info"), "mapfiles");
864 return fs.exists(mapfiles);
865 }
866
867
868
869
870
871
872
873
874
875
876
877
878 public static boolean isMajorCompactedPre020(final FileSystem fs,
879 final Path hbaseRootDir)
880 throws IOException {
881
882 FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, new DirFilter(fs));
883 for (FileStatus tableDir : tableDirs) {
884
885
886
887
888 Path d = tableDir.getPath();
889 if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
890 continue;
891 }
892 FileStatus[] regionDirs = fs.listStatus(d, new DirFilter(fs));
893 for (FileStatus regionDir : regionDirs) {
894 Path dd = regionDir.getPath();
895 if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
896 continue;
897 }
898
899 FileStatus[] familyDirs = fs.listStatus(dd, new DirFilter(fs));
900 for (FileStatus familyDir : familyDirs) {
901 Path family = familyDir.getPath();
902 FileStatus[] infoAndMapfile = fs.listStatus(family);
903
904 if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
905 LOG.debug(family.toString() +
906 " has more than just info and mapfile: " + infoAndMapfile.length);
907 return false;
908 }
909
910 for (int ll = 0; ll < 2; ll++) {
911 if (infoAndMapfile[ll].getPath().getName().equals("info") ||
912 infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
913 continue;
914 LOG.debug("Unexpected directory name: " +
915 infoAndMapfile[ll].getPath());
916 return false;
917 }
918
919
920 FileStatus[] familyStatus =
921 fs.listStatus(new Path(family, "mapfiles"));
922 if (familyStatus.length > 1) {
923 LOG.debug(family.toString() + " has " + familyStatus.length +
924 " files.");
925 return false;
926 }
927 }
928 }
929 }
930 return true;
931 }
932
933
934
935
936 static class FileFilter implements PathFilter {
937 private final FileSystem fs;
938
939 public FileFilter(final FileSystem fs) {
940 this.fs = fs;
941 }
942
943 @Override
944 public boolean accept(Path p) {
945 try {
946 return fs.isFile(p);
947 } catch (IOException e) {
948 LOG.debug("unable to verify if path=" + p + " is a regular file", e);
949 return false;
950 }
951 }
952 }
953
954
955
956
957 public static class DirFilter implements PathFilter {
958 private final FileSystem fs;
959
960 public DirFilter(final FileSystem fs) {
961 this.fs = fs;
962 }
963
964 @Override
965 public boolean accept(Path p) {
966 boolean isValid = false;
967 try {
968 if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(p)) {
969 isValid = false;
970 } else {
971 isValid = this.fs.getFileStatus(p).isDir();
972 }
973 } catch (IOException e) {
974 e.printStackTrace();
975 }
976 return isValid;
977 }
978 }
979
980
981
982
983
984
985
986
987 public static boolean isAppendSupported(final Configuration conf) {
988 boolean append = conf.getBoolean("dfs.support.append", false);
989 if (append) {
990 try {
991
992
993
994 SequenceFile.Writer.class.getMethod("syncFs", new Class<?> []{});
995 append = true;
996 } catch (SecurityException e) {
997 } catch (NoSuchMethodException e) {
998 append = false;
999 }
1000 }
1001 if (!append) {
1002
1003 try {
1004 FSDataOutputStream.class.getMethod("hflush", new Class<?> []{});
1005 append = true;
1006 } catch (NoSuchMethodException e) {
1007 append = false;
1008 }
1009 }
1010 return append;
1011 }
1012
1013
1014
1015
1016
1017
1018 public static boolean isHDFS(final Configuration conf) throws IOException {
1019 FileSystem fs = FileSystem.get(conf);
1020 String scheme = fs.getUri().getScheme();
1021 return scheme.equalsIgnoreCase("hdfs");
1022 }
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032 public abstract void recoverFileLease(final FileSystem fs, final Path p,
1033 Configuration conf) throws IOException;
1034
1035
1036
1037
1038
1039
1040
1041
1042 public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
1043 throws IOException {
1044
1045 FileStatus [] dirs = fs.listStatus(rootdir, new DirFilter(fs));
1046 List<Path> tabledirs = new ArrayList<Path>(dirs.length);
1047 for (FileStatus dir: dirs) {
1048 Path p = dir.getPath();
1049 String tableName = p.getName();
1050 if (!HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tableName)) {
1051 tabledirs.add(p);
1052 }
1053 }
1054 return tabledirs;
1055 }
1056
1057 public static Path getTablePath(Path rootdir, byte [] tableName) {
1058 return getTablePath(rootdir, Bytes.toString(tableName));
1059 }
1060
1061 public static Path getTablePath(Path rootdir, final String tableName) {
1062 return new Path(rootdir, tableName);
1063 }
1064
1065
1066
1067
1068 public static class RegionDirFilter implements PathFilter {
1069
1070 final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");
1071 final FileSystem fs;
1072
1073 public RegionDirFilter(FileSystem fs) {
1074 this.fs = fs;
1075 }
1076
1077 @Override
1078 public boolean accept(Path rd) {
1079 if (!regionDirPattern.matcher(rd.getName()).matches()) {
1080 return false;
1081 }
1082
1083 try {
1084 return fs.getFileStatus(rd).isDir();
1085 } catch (IOException ioe) {
1086
1087 LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1088 return false;
1089 }
1090 }
1091 }
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101 public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException {
1102
1103 FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs));
1104 List<Path> regionDirs = new ArrayList<Path>(rds.length);
1105 for (FileStatus rdfs: rds) {
1106 Path rdPath = rdfs.getPath();
1107 regionDirs.add(rdPath);
1108 }
1109 return regionDirs;
1110 }
1111
1112
1113
1114
1115
1116 public static class FamilyDirFilter implements PathFilter {
1117 final FileSystem fs;
1118
1119 public FamilyDirFilter(FileSystem fs) {
1120 this.fs = fs;
1121 }
1122
1123 @Override
1124 public boolean accept(Path rd) {
1125 try {
1126
1127 HColumnDescriptor.isLegalFamilyName(Bytes.toBytes(rd.getName()));
1128 } catch (IllegalArgumentException iae) {
1129
1130 return false;
1131 }
1132
1133 try {
1134 return fs.getFileStatus(rd).isDir();
1135 } catch (IOException ioe) {
1136
1137 LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1138 return false;
1139 }
1140 }
1141 }
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151 public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
1152
1153 FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
1154 List<Path> familyDirs = new ArrayList<Path>(fds.length);
1155 for (FileStatus fdfs: fds) {
1156 Path fdPath = fdfs.getPath();
1157 familyDirs.add(fdPath);
1158 }
1159 return familyDirs;
1160 }
1161
1162
1163
1164
1165 public static class HFileFilter implements PathFilter {
1166
1167 final public static Pattern hfilePattern = Pattern.compile("^([0-9a-f]+)$");
1168
1169 final FileSystem fs;
1170
1171 public HFileFilter(FileSystem fs) {
1172 this.fs = fs;
1173 }
1174
1175 @Override
1176 public boolean accept(Path rd) {
1177 if (!hfilePattern.matcher(rd.getName()).matches()) {
1178 return false;
1179 }
1180
1181 try {
1182
1183 return !fs.getFileStatus(rd).isDir();
1184 } catch (IOException ioe) {
1185
1186 LOG.warn("Skipping file " + rd +" due to IOException", ioe);
1187 return false;
1188 }
1189 }
1190 }
1191
1192
1193
1194
1195
1196
1197 public static FileSystem getCurrentFileSystem(Configuration conf)
1198 throws IOException {
1199 return getRootDir(conf).getFileSystem(conf);
1200 }
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215 public static Map<String, Path> getTableStoreFilePathMap(
1216 final FileSystem fs, final Path hbaseRootDir)
1217 throws IOException {
1218 Map<String, Path> map = new HashMap<String, Path>();
1219
1220
1221
1222
1223 DirFilter df = new DirFilter(fs);
1224
1225 FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
1226 for (FileStatus tableDir : tableDirs) {
1227
1228
1229
1230 Path d = tableDir.getPath();
1231 if (HConstants.HBASE_NON_TABLE_DIRS.contains(d.getName())) {
1232 continue;
1233 }
1234 FileStatus[] regionDirs = fs.listStatus(d, df);
1235 for (FileStatus regionDir : regionDirs) {
1236 Path dd = regionDir.getPath();
1237 if (dd.getName().equals(HConstants.HREGION_COMPACTIONDIR_NAME)) {
1238 continue;
1239 }
1240
1241 FileStatus[] familyDirs = fs.listStatus(dd, df);
1242 for (FileStatus familyDir : familyDirs) {
1243 Path family = familyDir.getPath();
1244
1245
1246 FileStatus[] familyStatus = fs.listStatus(family);
1247 for (FileStatus sfStatus : familyStatus) {
1248 Path sf = sfStatus.getPath();
1249 map.put( sf.getName(), sf);
1250 }
1251
1252 }
1253 }
1254 }
1255 return map;
1256 }
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267 public static FileStatus [] listStatus(final FileSystem fs,
1268 final Path dir, final PathFilter filter) throws IOException {
1269 FileStatus [] status = null;
1270 try {
1271 status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
1272 } catch (FileNotFoundException fnfe) {
1273
1274 LOG.debug(dir + " doesn't exist");
1275 }
1276 if (status == null || status.length < 1) return null;
1277 return status;
1278 }
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288 public static FileStatus[] listStatus(final FileSystem fs, final Path dir) throws IOException {
1289 return listStatus(fs, dir, null);
1290 }
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301 public static boolean delete(final FileSystem fs, final Path path, final boolean recursive)
1302 throws IOException {
1303 return fs.delete(path, recursive);
1304 }
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316 public static void checkAccess(User user, FileStatus file,
1317 FsAction action) throws AccessControlException {
1318
1319 String username = user.getShortName();
1320 if (username.equals(file.getOwner())) {
1321 if (file.getPermission().getUserAction().implies(action)) {
1322 return;
1323 }
1324 } else if (contains(user.getGroupNames(), file.getGroup())) {
1325 if (file.getPermission().getGroupAction().implies(action)) {
1326 return;
1327 }
1328 } else if (file.getPermission().getOtherAction().implies(action)) {
1329 return;
1330 }
1331 throw new AccessControlException("Permission denied:" + " action=" + action
1332 + " path=" + file.getPath() + " user=" + username);
1333 }
1334
1335 private static boolean contains(String[] groups, String user) {
1336 for (String group : groups) {
1337 if (group.equals(user)) {
1338 return true;
1339 }
1340 }
1341 return false;
1342 }
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352 public static boolean isExists(final FileSystem fs, final Path path) throws IOException {
1353 return fs.exists(path);
1354 }
1355
1356
1357
1358
1359
1360
1361
1362
1363 public static void logFileSystemState(final FileSystem fs, final Path root, Log LOG)
1364 throws IOException {
1365 LOG.debug("Current file system:");
1366 logFSTree(LOG, fs, root, "|-");
1367 }
1368
1369
1370
1371
1372
1373 private static void logFSTree(Log LOG, final FileSystem fs, final Path root, String prefix)
1374 throws IOException {
1375 FileStatus[] files = FSUtils.listStatus(fs, root, null);
1376 if (files == null) return;
1377
1378 for (FileStatus file : files) {
1379 if (file.isDir()) {
1380 LOG.debug(prefix + file.getPath().getName() + "/");
1381 logFSTree(LOG, fs, file.getPath(), prefix + "---");
1382 } else {
1383 LOG.debug(prefix + file.getPath().getName());
1384 }
1385 }
1386 }
1387 }