1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import java.io.IOException;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.FSDataOutputStream;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.fs.permission.FsPermission;
29 import org.apache.hadoop.hbase.regionserver.wal.HLogFileSystem;
30 import org.apache.hadoop.hbase.util.Threads;
31
32
33
34
35
36
37 public abstract class HBaseFileSystem {
38
39 public static final Log LOG = LogFactory.getLog(HBaseFileSystem.class);
40
41
42
43
44
45 protected static int hdfsClientRetriesNumber;
46 private static int baseSleepBeforeRetries;
47 private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
48 private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
49
50
51
52 static {
53 setRetryCounts(HBaseConfiguration.create());
54 }
55
56
57
58
59
60
61
62
63 public static boolean deleteFileFromFileSystem(FileSystem fs, Path dir)
64 throws IOException {
65 IOException lastIOE = null;
66 int i = 0;
67 do {
68 try {
69 return fs.delete(dir, false);
70 } catch (IOException ioe) {
71 lastIOE = ioe;
72 if (!fs.exists(dir)) return true;
73
74 sleepBeforeRetry("Delete File", i + 1);
75 }
76 } while (++i <= hdfsClientRetriesNumber);
77 throw new IOException("Exception in deleteFileFromFileSystem", lastIOE);
78 }
79
80
81
82
83
84
85
86
87
88 public static boolean deleteDirFromFileSystem(FileSystem fs, Path dir)
89 throws IOException {
90 IOException lastIOE = null;
91 int i = 0;
92 do {
93 try {
94 return fs.delete(dir, true);
95 } catch (IOException ioe) {
96 lastIOE = ioe;
97 if (!fs.exists(dir)) return true;
98
99 sleepBeforeRetry("Delete Dir", i + 1);
100 }
101 } while (++i <= hdfsClientRetriesNumber);
102 throw new IOException("Exception in deleteDirFromFileSystem", lastIOE);
103 }
104
105 protected static void setRetryCounts(Configuration conf) {
106 hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
107 DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
108 baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
109 DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
110 }
111
112
113
114
115
116
117
118
119
120
121 public static boolean makeDirOnFileSystem(FileSystem fs, Path dir)
122 throws IOException {
123 int i = 0;
124 IOException lastIOE = null;
125 do {
126 try {
127 return fs.mkdirs(dir);
128 } catch (IOException ioe) {
129 lastIOE = ioe;
130 if (fs.exists(dir)) return true;
131 sleepBeforeRetry("Create Directory", i+1);
132 }
133 } while (++i <= hdfsClientRetriesNumber);
134 throw new IOException("Exception in makeDirOnFileSystem", lastIOE);
135 }
136
137
138
139
140
141
142
143
144
145 public static boolean renameDirForFileSystem(FileSystem fs, Path src, Path dst)
146 throws IOException {
147 IOException lastIOE = null;
148 int i = 0;
149 do {
150 try {
151 return fs.rename(src, dst);
152 } catch (IOException ioe) {
153 lastIOE = ioe;
154 if (!fs.exists(src) && fs.exists(dst)) return true;
155
156 sleepBeforeRetry("Rename Directory", i + 1);
157 }
158 } while (++i <= hdfsClientRetriesNumber);
159 throw new IOException("Exception in renameDirForFileSystem", lastIOE);
160 }
161
162
163
164
165
166
167
168
169
170
171
172
173 public static FSDataOutputStream createPathOnFileSystem(FileSystem fs, Path dir,
174 boolean overwrite) throws IOException {
175 int i = 0;
176 boolean existsBefore = fs.exists(dir);
177 IOException lastIOE = null;
178 do {
179 try {
180 return fs.create(dir, overwrite);
181 } catch (IOException ioe) {
182 lastIOE = ioe;
183 if (existsBefore && !overwrite) throw ioe;
184 sleepBeforeRetry("Create Path", i + 1);
185 }
186 } while (++i <= hdfsClientRetriesNumber);
187 throw new IOException("Exception in createPathOnFileSystem", lastIOE);
188 }
189
190
191
192
193
194
195
196
197
198
199
200
201 public static FSDataOutputStream createPathWithPermsOnFileSystem(FileSystem fs,
202 Path path, FsPermission perm, boolean overwrite) throws IOException {
203 int i = 0;
204 IOException lastIOE = null;
205 boolean existsBefore = fs.exists(path);
206 do {
207 try {
208 return fs.create(path, perm, overwrite, fs.getConf().getInt("io.file.buffer.size", 4096),
209 fs.getDefaultReplication(), fs.getDefaultBlockSize(), null);
210 } catch (IOException ioe) {
211 lastIOE = ioe;
212 if (existsBefore && !overwrite) throw ioe;
213 sleepBeforeRetry("Create Path with Perms", i + 1);
214 }
215 } while (++i <= hdfsClientRetriesNumber);
216 throw new IOException("Exception in createPathWithPermsOnFileSystem", lastIOE);
217 }
218
219
220
221
222
223
224
225
226 public static boolean createNewFileOnFileSystem(FileSystem fs, Path file)
227 throws IOException {
228 int i = 0;
229 IOException lastIOE = null;
230 do {
231 try {
232 return fs.createNewFile(file);
233 } catch (IOException ioe) {
234 lastIOE = ioe;
235 if (fs.exists(file)) return true;
236 sleepBeforeRetry("Create NewFile", i + 1);
237 }
238 } while (++i <= hdfsClientRetriesNumber);
239 throw new IOException("Exception in createNewFileOnFileSystem", lastIOE);
240 }
241
242
243
244
245
246 protected static void sleepBeforeRetry(String msg, int sleepMultiplier) {
247 if (sleepMultiplier > hdfsClientRetriesNumber) {
248 LOG.warn(msg + ", retries exhausted");
249 return;
250 }
251 LOG.info(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
252 Threads.sleep(baseSleepBeforeRetries * sleepMultiplier);
253 }
254 }