1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import java.io.IOException;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.FSDataOutputStream;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.fs.permission.FsPermission;
29 import org.apache.hadoop.hbase.regionserver.wal.HLogFileSystem;
30 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
31 import org.apache.hadoop.hbase.util.FSUtils;
32 import org.apache.hadoop.hbase.util.Threads;
33
34
35
36
37
38
39 public abstract class HBaseFileSystem {
40
41 public static final Log LOG = LogFactory.getLog(HBaseFileSystem.class);
42
43
44
45
46
47 protected static int hdfsClientRetriesNumber;
48 private static int baseSleepBeforeRetries;
49 private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
50 private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
51
52
53
54 static {
55 setRetryCounts(HBaseConfiguration.create());
56 }
57
58
59
60
61
62
63
64
65 public static boolean deleteFileFromFileSystem(FileSystem fs, Path file)
66 throws IOException {
67 IOException lastIOE = null;
68 int i = 0;
69 do {
70 try {
71 return fs.delete(file, false);
72 } catch (IOException ioe) {
73 lastIOE = ioe;
74 if (!fs.exists(file)) return true;
75
76 sleepBeforeRetry("Delete File", i + 1);
77 }
78 } while (++i <= hdfsClientRetriesNumber);
79 throw new IOException("Exception in deleteFileFromFileSystem", lastIOE);
80 }
81
82
83
84
85
86
87
88
89
90 public static boolean deleteDirFromFileSystem(FileSystem fs, Path dir)
91 throws IOException {
92 IOException lastIOE = null;
93 int i = 0;
94 do {
95 try {
96 return fs.delete(dir, true);
97 } catch (IOException ioe) {
98 lastIOE = ioe;
99 if (!fs.exists(dir)) return true;
100
101 sleepBeforeRetry("Delete Dir", i + 1);
102 }
103 } while (++i <= hdfsClientRetriesNumber);
104 throw new IOException("Exception in deleteDirFromFileSystem", lastIOE);
105 }
106
107 protected static void setRetryCounts(Configuration conf) {
108 hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
109 DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
110 baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
111 DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
112 }
113
114
115
116
117
118
119
120
121
122
123 public static boolean makeDirOnFileSystem(FileSystem fs, Path dir)
124 throws IOException {
125 int i = 0;
126 IOException lastIOE = null;
127 do {
128 try {
129 return fs.mkdirs(dir);
130 } catch (IOException ioe) {
131 lastIOE = ioe;
132 if (fs.exists(dir)) return true;
133 sleepBeforeRetry("Create Directory", i+1);
134 }
135 } while (++i <= hdfsClientRetriesNumber);
136 throw new IOException("Exception in makeDirOnFileSystem", lastIOE);
137 }
138
139
140
141
142
143
144
145
146
147 public static boolean renameDirForFileSystem(FileSystem fs, Path src, Path dst)
148 throws IOException {
149 IOException lastIOE = null;
150 int i = 0;
151 do {
152 try {
153 return fs.rename(src, dst);
154 } catch (IOException ioe) {
155 lastIOE = ioe;
156 if (!fs.exists(src) && fs.exists(dst)) return true;
157
158 sleepBeforeRetry("Rename Directory", i + 1);
159 }
160 } while (++i <= hdfsClientRetriesNumber);
161 throw new IOException("Exception in renameDirForFileSystem", lastIOE);
162 }
163
164
165
166
167
168
169
170
171
172
173
174
175 public static FSDataOutputStream createPathOnFileSystem(FileSystem fs, Path dir,
176 boolean overwrite) throws IOException {
177 int i = 0;
178 boolean existsBefore = fs.exists(dir);
179 IOException lastIOE = null;
180 do {
181 try {
182 return fs.create(dir, overwrite);
183 } catch (IOException ioe) {
184 lastIOE = ioe;
185 if (existsBefore && !overwrite) throw ioe;
186 sleepBeforeRetry("Create Path", i + 1);
187 }
188 } while (++i <= hdfsClientRetriesNumber);
189 throw new IOException("Exception in createPathOnFileSystem", lastIOE);
190 }
191
192
193
194
195
196
197
198
199
200
201
202
203 public static FSDataOutputStream createPathWithPermsOnFileSystem(FileSystem fs,
204 Path path, FsPermission perm, boolean overwrite) throws IOException {
205 int i = 0;
206 IOException lastIOE = null;
207 boolean existsBefore = fs.exists(path);
208 do {
209 try {
210 return fs.create(path, perm, overwrite, FSUtils.getDefaultBufferSize(fs),
211 FSUtils.getDefaultReplication(fs, path), FSUtils.getDefaultBlockSize(fs, path), null);
212 } catch (IOException ioe) {
213 lastIOE = ioe;
214 if (existsBefore && !overwrite) throw ioe;
215 sleepBeforeRetry("Create Path with Perms", i + 1);
216 }
217 } while (++i <= hdfsClientRetriesNumber);
218 throw new IOException("Exception in createPathWithPermsOnFileSystem", lastIOE);
219 }
220
221
222
223
224
225
226
227
228 public static boolean createNewFileOnFileSystem(FileSystem fs, Path file)
229 throws IOException {
230 int i = 0;
231 IOException lastIOE = null;
232 do {
233 try {
234 return fs.createNewFile(file);
235 } catch (IOException ioe) {
236 lastIOE = ioe;
237 if (fs.exists(file)) return true;
238 sleepBeforeRetry("Create NewFile", i + 1);
239 }
240 } while (++i <= hdfsClientRetriesNumber);
241 throw new IOException("Exception in createNewFileOnFileSystem", lastIOE);
242 }
243
244
245
246
247
248 protected static void sleepBeforeRetry(String msg, int sleepMultiplier) {
249 if (sleepMultiplier > hdfsClientRetriesNumber) {
250 LOG.warn(msg + ", retries exhausted");
251 return;
252 }
253 LOG.info(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
254 Threads.sleep(baseSleepBeforeRetries * sleepMultiplier);
255 }
256
257
258
259
260 public static boolean renameAndSetModifyTime(final FileSystem fs, Path src, Path dest)
261 throws IOException {
262
263 fs.setTimes(src, EnvironmentEdgeManager.currentTimeMillis(), -1);
264 return renameDirForFileSystem(fs, src, dest);
265 }
266 }