1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.FileNotFoundException;
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.UUID;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FSDataInputStream;
35 import org.apache.hadoop.fs.FSDataOutputStream;
36 import org.apache.hadoop.fs.FileStatus;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.FileUtil;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.fs.PathFilter;
41 import org.apache.hadoop.fs.permission.FsPermission;
42 import org.apache.hadoop.hbase.HColumnDescriptor;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HRegionInfo;
45 import org.apache.hadoop.hbase.HTableDescriptor;
46 import org.apache.hadoop.hbase.KeyValue;
47 import org.apache.hadoop.hbase.backup.HFileArchiver;
48 import org.apache.hadoop.hbase.fs.HFileSystem;
49 import org.apache.hadoop.hbase.io.Reference;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.FSHDFSUtils;
52 import org.apache.hadoop.hbase.util.FSUtils;
53 import org.apache.hadoop.hbase.util.Threads;
54
55
56
57
58
59 @InterfaceAudience.Private
60 public class HRegionFileSystem {
61 public static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
62
63
64 public final static String REGION_INFO_FILE = ".regioninfo";
65
66
67 public static final String REGION_MERGES_DIR = ".merges";
68
69
70 public static final String REGION_SPLITS_DIR = ".splits";
71
72
73 private static final String REGION_TEMP_DIR = ".tmp";
74
75 private final HRegionInfo regionInfo;
76 private final Configuration conf;
77 private final Path tableDir;
78 private final FileSystem fs;
79
80
81
82
83
84 private final int hdfsClientRetriesNumber;
85 private final int baseSleepBeforeRetries;
86 private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
87 private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
88
89
90
91
92
93
94
95
96 HRegionFileSystem(final Configuration conf, final FileSystem fs, final Path tableDir,
97 final HRegionInfo regionInfo) {
98 this.fs = fs;
99 this.conf = conf;
100 this.tableDir = tableDir;
101 this.regionInfo = regionInfo;
102 this.hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
103 DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
104 this.baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
105 DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
106 }
107
108
109 public FileSystem getFileSystem() {
110 return this.fs;
111 }
112
113
114 public HRegionInfo getRegionInfo() {
115 return this.regionInfo;
116 }
117
118
119 public Path getTableDir() {
120 return this.tableDir;
121 }
122
123
124 public Path getRegionDir() {
125 return new Path(this.tableDir, this.regionInfo.getEncodedName());
126 }
127
128
129
130
131
132 Path getTempDir() {
133 return new Path(getRegionDir(), REGION_TEMP_DIR);
134 }
135
136
137
138
139 void cleanupTempDir() throws IOException {
140 deleteDir(getTempDir());
141 }
142
143
144
145
146
147
148
149
150
151 public Path getStoreDir(final String familyName) {
152 return new Path(this.getRegionDir(), familyName);
153 }
154
155
156
157
158
159
160
161 Path createStoreDir(final String familyName) throws IOException {
162 Path storeDir = getStoreDir(familyName);
163 if(!fs.exists(storeDir) && !createDir(storeDir))
164 throw new IOException("Failed creating "+storeDir);
165 return storeDir;
166 }
167
168
169
170
171
172
173
174 public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
175 return getStoreFiles(Bytes.toString(familyName));
176 }
177
178 public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
179 return getStoreFiles(familyName, true);
180 }
181
182
183
184
185
186
187
188 public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
189 throws IOException {
190 Path familyDir = getStoreDir(familyName);
191 FileStatus[] files = FSUtils.listStatus(this.fs, familyDir);
192 if (files == null) {
193 LOG.debug("No StoreFiles for: " + familyDir);
194 return null;
195 }
196
197 ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
198 for (FileStatus status: files) {
199 if (validate && !StoreFileInfo.isValid(status)) {
200 LOG.warn("Invalid StoreFile: " + status.getPath());
201 continue;
202 }
203
204 storeFiles.add(new StoreFileInfo(this.conf, this.fs, status));
205 }
206 return storeFiles;
207 }
208
209
210
211
212
213
214
215
216 Path getStoreFilePath(final String familyName, final String fileName) {
217 Path familyDir = getStoreDir(familyName);
218 return new Path(familyDir, fileName).makeQualified(this.fs);
219 }
220
221
222
223
224
225
226
227
228 StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
229 throws IOException {
230 Path familyDir = getStoreDir(familyName);
231 FileStatus status = fs.getFileStatus(new Path(familyDir, fileName));
232 return new StoreFileInfo(this.conf, this.fs, status);
233 }
234
235
236
237
238
239
240
241 public boolean hasReferences(final String familyName) throws IOException {
242 FileStatus[] files = FSUtils.listStatus(fs, getStoreDir(familyName),
243 new PathFilter () {
244 public boolean accept(Path path) {
245 return StoreFileInfo.isReference(path);
246 }
247 }
248 );
249 return files != null && files.length > 0;
250 }
251
252
253
254
255
256
257
258 public boolean hasReferences(final HTableDescriptor htd) throws IOException {
259 for (HColumnDescriptor family : htd.getFamilies()) {
260 if (hasReferences(family.getNameAsString())) {
261 return true;
262 }
263 }
264 return false;
265 }
266
267
268
269
270
271 public Collection<String> getFamilies() throws IOException {
272 FileStatus[] fds = FSUtils.listStatus(fs, getRegionDir(), new FSUtils.FamilyDirFilter(fs));
273 if (fds == null) return null;
274
275 ArrayList<String> families = new ArrayList<String>(fds.length);
276 for (FileStatus status: fds) {
277 families.add(status.getPath().getName());
278 }
279
280 return families;
281 }
282
283
284
285
286
287
288 public void deleteFamily(final String familyName) throws IOException {
289
290 HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, Bytes.toBytes(familyName));
291
292
293 Path familyDir = getStoreDir(familyName);
294 if(fs.exists(familyDir) && !deleteDir(familyDir))
295 throw new IOException("Could not delete family " + familyName
296 + " from FileSystem for region " + regionInfo.getRegionNameAsString() + "("
297 + regionInfo.getEncodedName() + ")");
298 }
299
300
301
302
303
304
305 private static String generateUniqueName(final String suffix) {
306 String name = UUID.randomUUID().toString().replaceAll("-", "");
307 if (suffix != null) name += suffix;
308 return name;
309 }
310
311
312
313
314
315
316
317
318
319
320
321
322 public Path createTempName() {
323 return createTempName(null);
324 }
325
326
327
328
329
330
331
332
333
334
335
336
337
338 public Path createTempName(final String suffix) {
339 return new Path(getTempDir(), generateUniqueName(suffix));
340 }
341
342
343
344
345
346
347
348
349 public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
350 return commitStoreFile(familyName, buildPath, -1, false);
351 }
352
353
354
355
356
357
358
359
360
361
362 private Path commitStoreFile(final String familyName, final Path buildPath,
363 final long seqNum, final boolean generateNewName) throws IOException {
364 Path storeDir = getStoreDir(familyName);
365 if(!fs.exists(storeDir) && !createDir(storeDir))
366 throw new IOException("Failed creating " + storeDir);
367
368 String name = buildPath.getName();
369 if (generateNewName) {
370 name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
371 }
372 Path dstPath = new Path(storeDir, name);
373 if (!fs.exists(buildPath)) {
374 throw new FileNotFoundException(buildPath.toString());
375 }
376 LOG.debug("Committing store file " + buildPath + " as " + dstPath);
377
378 if (!rename(buildPath, dstPath)) {
379 throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
380 }
381 return dstPath;
382 }
383
384
385
386
387
388
389
390 void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
391 for (Map.Entry<byte[], List<StoreFile>> es: storeFiles.entrySet()) {
392 String familyName = Bytes.toString(es.getKey());
393 for (StoreFile sf: es.getValue()) {
394 commitStoreFile(familyName, sf.getPath());
395 }
396 }
397 }
398
399
400
401
402
403
404
405 public void removeStoreFile(final String familyName, final Path filePath)
406 throws IOException {
407 HFileArchiver.archiveStoreFile(this.conf, this.fs, this.regionInfo,
408 this.tableDir, Bytes.toBytes(familyName), filePath);
409 }
410
411
412
413
414
415
416
417 public void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
418 throws IOException {
419 HFileArchiver.archiveStoreFiles(this.conf, this.fs, this.regionInfo,
420 this.tableDir, Bytes.toBytes(familyName), storeFiles);
421 }
422
423
424
425
426
427
428
429
430
431
432
433
434 Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
435 throws IOException {
436
437 FileSystem srcFs = srcPath.getFileSystem(conf);
438 FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
439
440
441
442
443 if (!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)) {
444 LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " +
445 "the destination store. Copying file over to destination filesystem.");
446 Path tmpPath = createTempName();
447 FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, conf);
448 LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
449 srcPath = tmpPath;
450 }
451
452 return commitStoreFile(familyName, srcPath, seqNum, true);
453 }
454
455
456
457
458
459 Path getSplitsDir() {
460 return new Path(getRegionDir(), REGION_SPLITS_DIR);
461 }
462
463 Path getSplitsDir(final HRegionInfo hri) {
464 return new Path(getSplitsDir(), hri.getEncodedName());
465 }
466
467
468
469
470 void cleanupSplitsDir() throws IOException {
471 deleteDir(getSplitsDir());
472 }
473
474
475
476
477
478
479
480 void cleanupAnySplitDetritus() throws IOException {
481 Path splitdir = this.getSplitsDir();
482 if (!fs.exists(splitdir)) return;
483
484
485
486
487
488
489
490 FileStatus[] daughters = FSUtils.listStatus(fs, splitdir, new FSUtils.DirFilter(fs));
491 if (daughters != null) {
492 for (FileStatus daughter: daughters) {
493 Path daughterDir = new Path(getTableDir(), daughter.getPath().getName());
494 if (fs.exists(daughterDir) && !deleteDir(daughterDir)) {
495 throw new IOException("Failed delete of " + daughterDir);
496 }
497 }
498 }
499 cleanupSplitsDir();
500 LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
501 }
502
503
504
505
506
507
508 void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
509 Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
510 if (this.fs.exists(regionDir) && !deleteDir(regionDir)) {
511 throw new IOException("Failed delete of " + regionDir);
512 }
513 }
514
515
516
517
518
519
520
521 Path commitDaughterRegion(final HRegionInfo regionInfo) throws IOException {
522 Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
523 Path daughterTmpDir = this.getSplitsDir(regionInfo);
524 if (fs.exists(daughterTmpDir)) {
525
526 Path regionInfoFile = new Path(daughterTmpDir, REGION_INFO_FILE);
527 byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
528 writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
529
530
531 if (!rename(daughterTmpDir, regionDir)) {
532 throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir);
533 }
534 }
535 return regionDir;
536 }
537
538
539
540
541 void createSplitsDir() throws IOException {
542 Path splitdir = getSplitsDir();
543 if (fs.exists(splitdir)) {
544 LOG.info("The " + splitdir + " directory exists. Hence deleting it to recreate it");
545 if (!deleteDir(splitdir)) {
546 throw new IOException("Failed deletion of " + splitdir
547 + " before creating them again.");
548 }
549 }
550
551 if (!createDir(splitdir)) {
552 throw new IOException("Failed create of " + splitdir);
553 }
554 }
555
556
557
558
559
560
561
562
563
564
565
566
567 Path splitStoreFile(final HRegionInfo hri, final String familyName,
568 final StoreFile f, final byte[] splitRow, final boolean top) throws IOException {
569
570
571
572 if (top) {
573
574 KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
575 byte[] lastKey = f.createReader().getLastKey();
576
577 if (lastKey == null) return null;
578 if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
579 splitKey.getKeyOffset(), splitKey.getKeyLength(), lastKey, 0, lastKey.length) > 0) {
580 return null;
581 }
582 } else {
583
584 KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
585 byte[] firstKey = f.createReader().getFirstKey();
586
587 if (firstKey == null) return null;
588 if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
589 splitKey.getKeyOffset(), splitKey.getKeyLength(), firstKey, 0, firstKey.length) < 0) {
590 return null;
591 }
592 }
593
594 f.getReader().close(true);
595
596 Path splitDir = new Path(getSplitsDir(hri), familyName);
597
598 Reference r =
599 top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
600
601
602
603
604 String parentRegionName = regionInfo.getEncodedName();
605
606
607 Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
608 return r.write(fs, p);
609 }
610
611
612
613
614
615 Path getMergesDir() {
616 return new Path(getRegionDir(), REGION_MERGES_DIR);
617 }
618
619 Path getMergesDir(final HRegionInfo hri) {
620 return new Path(getMergesDir(), hri.getEncodedName());
621 }
622
623
624
625
626 void cleanupMergesDir() throws IOException {
627 deleteDir(getMergesDir());
628 }
629
630
631
632
633
634
635 void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
636 Path regionDir = new Path(this.tableDir, mergedRegion.getEncodedName());
637 if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
638 throw new IOException("Failed delete of " + regionDir);
639 }
640 }
641
642
643
644
645
646
647 void createMergesDir() throws IOException {
648 Path mergesdir = getMergesDir();
649 if (fs.exists(mergesdir)) {
650 LOG.info("The " + mergesdir
651 + " directory exists. Hence deleting it to recreate it");
652 if (!fs.delete(mergesdir, true)) {
653 throw new IOException("Failed deletion of " + mergesdir
654 + " before creating them again.");
655 }
656 }
657 if (!fs.mkdirs(mergesdir))
658 throw new IOException("Failed create of " + mergesdir);
659 }
660
661
662
663
664
665
666
667
668
669
670
671 Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
672 final StoreFile f, final Path mergedDir)
673 throws IOException {
674 Path referenceDir = new Path(new Path(mergedDir,
675 mergedRegion.getEncodedName()), familyName);
676
677 Reference r = Reference.createTopReference(regionInfo.getStartKey());
678
679
680
681
682 String mergingRegionName = regionInfo.getEncodedName();
683
684
685 Path p = new Path(referenceDir, f.getPath().getName() + "."
686 + mergingRegionName);
687 return r.write(fs, p);
688 }
689
690
691
692
693
694
695
696 void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
697 Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
698 Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo);
699
700 if (mergedRegionTmpDir != null && fs.exists(mergedRegionTmpDir)) {
701 if (!fs.rename(mergedRegionTmpDir, regionDir)) {
702 throw new IOException("Unable to rename " + mergedRegionTmpDir + " to "
703 + regionDir);
704 }
705 }
706 }
707
708
709
710
711
712
713
714
715
716 void logFileSystemState(final Log LOG) throws IOException {
717 FSUtils.logFileSystemState(fs, this.getRegionDir(), LOG);
718 }
719
720
721
722
723
724
725 private static byte[] getRegionInfoFileContent(final HRegionInfo hri) throws IOException {
726 return hri.toDelimitedByteArray();
727 }
728
729
730
731
732
733
734
735
736 public static HRegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir)
737 throws IOException {
738 FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE));
739 try {
740 return HRegionInfo.parseFrom(in);
741 } finally {
742 in.close();
743 }
744 }
745
746
747
748
749 private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
750 final Path regionInfoFile, final byte[] content) throws IOException {
751
752 FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
753
754 FSDataOutputStream out = FSUtils.create(fs, regionInfoFile, perms, null);
755 try {
756 out.write(content);
757 } finally {
758 out.close();
759 }
760 }
761
762
763
764
765
766 void checkRegionInfoOnFilesystem() throws IOException {
767
768
769
770
771
772 byte[] content = getRegionInfoFileContent(regionInfo);
773 try {
774 Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
775
776 FileStatus status = fs.getFileStatus(regionInfoFile);
777 if (status != null && status.getLen() == content.length) {
778
779
780 return;
781 }
782
783 LOG.info("Rewriting .regioninfo file at: " + regionInfoFile);
784 if (!fs.delete(regionInfoFile, false)) {
785 throw new IOException("Unable to remove existing " + regionInfoFile);
786 }
787 } catch (FileNotFoundException e) {
788 LOG.warn(REGION_INFO_FILE + " file not found for region: " + regionInfo.getEncodedName());
789 }
790
791
792 writeRegionInfoOnFilesystem(content, true);
793 }
794
795
796
797
798
799 private void writeRegionInfoOnFilesystem(boolean useTempDir) throws IOException {
800 byte[] content = getRegionInfoFileContent(regionInfo);
801 writeRegionInfoOnFilesystem(content, useTempDir);
802 }
803
804
805
806
807
808
809 private void writeRegionInfoOnFilesystem(final byte[] regionInfoContent,
810 final boolean useTempDir) throws IOException {
811 Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
812 if (useTempDir) {
813
814
815
816
817
818
819 Path tmpPath = new Path(getTempDir(), REGION_INFO_FILE);
820
821
822
823
824
825 if (FSUtils.isExists(fs, tmpPath)) {
826 FSUtils.delete(fs, tmpPath, true);
827 }
828
829
830 writeRegionInfoFileContent(conf, fs, tmpPath, regionInfoContent);
831
832
833 if (fs.exists(tmpPath) && !rename(tmpPath, regionInfoFile)) {
834 throw new IOException("Unable to rename " + tmpPath + " to " + regionInfoFile);
835 }
836 } else {
837
838 writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
839 }
840 }
841
842
843
844
845
846
847
848
849
850 public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf,
851 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
852 HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
853 Path regionDir = regionFs.getRegionDir();
854
855 if (fs.exists(regionDir)) {
856 LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
857 throw new IOException("The specified region already exists on disk: " + regionDir);
858 }
859
860
861 if (!createDirOnFileSystem(fs, conf, regionDir)) {
862 LOG.warn("Unable to create the region directory: " + regionDir);
863 throw new IOException("Unable to create region directory: " + regionDir);
864 }
865
866
867 regionFs.writeRegionInfoOnFilesystem(false);
868 return regionFs;
869 }
870
871
872
873
874
875
876
877
878
879
880 public static HRegionFileSystem openRegionFromFileSystem(final Configuration conf,
881 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo, boolean readOnly)
882 throws IOException {
883 HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
884 Path regionDir = regionFs.getRegionDir();
885
886 if (!fs.exists(regionDir)) {
887 LOG.warn("Trying to open a region that do not exists on disk: " + regionDir);
888 throw new IOException("The specified region do not exists on disk: " + regionDir);
889 }
890
891 if (!readOnly) {
892
893 regionFs.cleanupTempDir();
894 regionFs.cleanupSplitsDir();
895 regionFs.cleanupMergesDir();
896
897
898 regionFs.checkRegionInfoOnFilesystem();
899 }
900
901 return regionFs;
902 }
903
904
905
906
907
908
909
910
911
912 public static void deleteRegionFromFileSystem(final Configuration conf,
913 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
914 HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
915 Path regionDir = regionFs.getRegionDir();
916
917 if (!fs.exists(regionDir)) {
918 LOG.warn("Trying to delete a region that do not exists on disk: " + regionDir);
919 return;
920 }
921
922 if (LOG.isDebugEnabled()) {
923 LOG.debug("DELETING region " + regionDir);
924 }
925
926
927 Path rootDir = FSUtils.getRootDir(conf);
928 HFileArchiver.archiveRegion(fs, rootDir, tableDir, regionDir);
929
930
931 if (!fs.delete(regionDir, true)) {
932 LOG.warn("Failed delete of " + regionDir);
933 }
934 }
935
936
937
938
939
940
941
942
943 boolean createDir(Path dir) throws IOException {
944 int i = 0;
945 IOException lastIOE = null;
946 do {
947 try {
948 return fs.mkdirs(dir);
949 } catch (IOException ioe) {
950 lastIOE = ioe;
951 if (fs.exists(dir)) return true;
952 sleepBeforeRetry("Create Directory", i+1);
953 }
954 } while (++i <= hdfsClientRetriesNumber);
955 throw new IOException("Exception in createDir", lastIOE);
956 }
957
958
959
960
961
962
963
964
965 boolean rename(Path srcpath, Path dstPath) throws IOException {
966 IOException lastIOE = null;
967 int i = 0;
968 do {
969 try {
970 return fs.rename(srcpath, dstPath);
971 } catch (IOException ioe) {
972 lastIOE = ioe;
973 if (!fs.exists(srcpath) && fs.exists(dstPath)) return true;
974
975 sleepBeforeRetry("Rename Directory", i+1);
976 }
977 } while (++i <= hdfsClientRetriesNumber);
978 throw new IOException("Exception in rename", lastIOE);
979 }
980
981
982
983
984
985
986
987 boolean deleteDir(Path dir) throws IOException {
988 IOException lastIOE = null;
989 int i = 0;
990 do {
991 try {
992 return fs.delete(dir, true);
993 } catch (IOException ioe) {
994 lastIOE = ioe;
995 if (!fs.exists(dir)) return true;
996
997 sleepBeforeRetry("Delete Directory", i+1);
998 }
999 } while (++i <= hdfsClientRetriesNumber);
1000 throw new IOException("Exception in DeleteDir", lastIOE);
1001 }
1002
1003
1004
1005
1006 private void sleepBeforeRetry(String msg, int sleepMultiplier) {
1007 sleepBeforeRetry(msg, sleepMultiplier, baseSleepBeforeRetries, hdfsClientRetriesNumber);
1008 }
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020 private static boolean createDirOnFileSystem(FileSystem fs, Configuration conf, Path dir)
1021 throws IOException {
1022 int i = 0;
1023 IOException lastIOE = null;
1024 int hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
1025 DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
1026 int baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
1027 DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
1028 do {
1029 try {
1030 return fs.mkdirs(dir);
1031 } catch (IOException ioe) {
1032 lastIOE = ioe;
1033 if (fs.exists(dir)) return true;
1034 sleepBeforeRetry("Create Directory", i+1, baseSleepBeforeRetries, hdfsClientRetriesNumber);
1035 }
1036 } while (++i <= hdfsClientRetriesNumber);
1037 throw new IOException("Exception in createDir", lastIOE);
1038 }
1039
1040
1041
1042
1043
1044 private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries,
1045 int hdfsClientRetriesNumber) {
1046 if (sleepMultiplier > hdfsClientRetriesNumber) {
1047 LOG.debug(msg + ", retries exhausted");
1048 return;
1049 }
1050 LOG.debug(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
1051 Threads.sleep((long)baseSleepBeforeRetries * sleepMultiplier);
1052 }
1053 }