1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.fs;
22
23 import java.io.Closeable;
24 import java.io.IOException;
25 import java.lang.reflect.Field;
26 import java.lang.reflect.InvocationHandler;
27 import java.lang.reflect.InvocationTargetException;
28 import java.lang.reflect.Method;
29 import java.lang.reflect.Modifier;
30 import java.lang.reflect.Proxy;
31 import java.lang.reflect.UndeclaredThrowableException;
32 import java.net.URI;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.FilterFileSystem;
40 import org.apache.hadoop.fs.LocalFileSystem;
41 import org.apache.hadoop.fs.Path;
42 import org.apache.hadoop.hbase.ServerName;
43 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
44 import org.apache.hadoop.hdfs.DFSClient;
45 import org.apache.hadoop.hdfs.DistributedFileSystem;
46 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
47 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
48 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
49 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
50 import org.apache.hadoop.ipc.RPC;
51 import org.apache.hadoop.util.Progressable;
52 import org.apache.hadoop.util.ReflectionUtils;
53
54
55
56
57
58
59
60
61 public class HFileSystem extends FilterFileSystem {
62 public static final Log LOG = LogFactory.getLog(HFileSystem.class);
63
64 private final FileSystem noChecksumFs;
65 private final boolean useHBaseChecksum;
66
67
68
69
70
71
72
73
74 public HFileSystem(Configuration conf, boolean useHBaseChecksum)
75 throws IOException {
76
77
78
79
80 this.fs = FileSystem.get(conf);
81 this.useHBaseChecksum = useHBaseChecksum;
82
83 fs.initialize(getDefaultUri(conf), conf);
84 addLocationsOrderInterceptor(conf);
85
86
87
88
89
90
91
92
93
94 if (useHBaseChecksum && !(fs instanceof LocalFileSystem)) {
95 conf = new Configuration(conf);
96 conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true);
97 this.noChecksumFs = newInstanceFileSystem(conf);
98 this.noChecksumFs.setVerifyChecksum(false);
99 } else {
100 this.noChecksumFs = fs;
101 }
102 }
103
104
105
106
107
108
109
110 public HFileSystem(FileSystem fs) {
111 this.fs = fs;
112 this.noChecksumFs = fs;
113 this.useHBaseChecksum = false;
114 }
115
116
117
118
119
120
121
122
123 public FileSystem getNoChecksumFs() {
124 return noChecksumFs;
125 }
126
127
128
129
130
131 public FileSystem getBackingFs() throws IOException {
132 return fs;
133 }
134
135
136
137
138
139
140 public boolean useHBaseChecksum() {
141 return useHBaseChecksum;
142 }
143
144
145
146
147 @Override
148 public void close() throws IOException {
149 super.close();
150 if (this.noChecksumFs != fs) {
151 this.noChecksumFs.close();
152 }
153 }
154
155
156
157
158
159
160
161
162
163 private static FileSystem newInstanceFileSystem(Configuration conf)
164 throws IOException {
165 URI uri = FileSystem.getDefaultUri(conf);
166 FileSystem fs = null;
167 Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
168 if (clazz != null) {
169
170 fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
171 fs.initialize(uri, conf);
172 } else {
173
174
175
176 Configuration clone = new Configuration(conf);
177 clone.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true);
178 fs = FileSystem.get(uri, clone);
179 }
180 if (fs == null) {
181 throw new IOException("No FileSystem for scheme: " + uri.getScheme());
182 }
183
184 return fs;
185 }
186
187 public static boolean addLocationsOrderInterceptor(Configuration conf) throws IOException {
188 return addLocationsOrderInterceptor(conf, new ReorderWALBlocks());
189 }
190
191
192
193
194
195
196
197
198
199 static boolean addLocationsOrderInterceptor(Configuration conf, final ReorderBlocks lrb) {
200 if (!conf.getBoolean("hbase.filesystem.reorder.blocks", true)) {
201 LOG.debug("addLocationsOrderInterceptor configured to false");
202 return false;
203 }
204
205 FileSystem fs;
206 try {
207 fs = FileSystem.get(conf);
208 } catch (IOException e) {
209 LOG.warn("Can't get the file system from the conf.", e);
210 return false;
211 }
212
213 if (!(fs instanceof DistributedFileSystem)) {
214 LOG.debug("The file system is not a DistributedFileSystem. " +
215 "Skipping on block location reordering");
216 return false;
217 }
218
219 DistributedFileSystem dfs = (DistributedFileSystem) fs;
220 DFSClient dfsc = dfs.getClient();
221 if (dfsc == null) {
222 LOG.warn("The DistributedFileSystem does not contain a DFSClient. Can't add the location " +
223 "block reordering interceptor. Continuing, but this is unexpected."
224 );
225 return false;
226 }
227
228 try {
229 Field nf = DFSClient.class.getDeclaredField("namenode");
230 nf.setAccessible(true);
231 Field modifiersField = Field.class.getDeclaredField("modifiers");
232 modifiersField.setAccessible(true);
233 modifiersField.setInt(nf, nf.getModifiers() & ~Modifier.FINAL);
234
235 ClientProtocol namenode = (ClientProtocol) nf.get(dfsc);
236 if (namenode == null) {
237 LOG.warn("The DFSClient is not linked to a namenode. Can't add the location block" +
238 " reordering interceptor. Continuing, but this is unexpected."
239 );
240 return false;
241 }
242
243 ClientProtocol cp1 = createReorderingProxy(namenode, lrb, conf);
244 nf.set(dfsc, cp1);
245 LOG.info("Added intercepting call to namenode#getBlockLocations so can do block reordering" +
246 " using class " + lrb.getClass());
247 } catch (NoSuchFieldException e) {
248 LOG.warn("Can't modify the DFSClient#namenode field to add the location reorder.", e);
249 return false;
250 } catch (IllegalAccessException e) {
251 LOG.warn("Can't modify the DFSClient#namenode field to add the location reorder.", e);
252 return false;
253 }
254
255 return true;
256 }
257
258 private static ClientProtocol createReorderingProxy(final ClientProtocol cp,
259 final ReorderBlocks lrb, final Configuration conf) {
260 return (ClientProtocol) Proxy.newProxyInstance
261 (cp.getClass().getClassLoader(),
262 new Class[]{ClientProtocol.class, Closeable.class},
263 new InvocationHandler() {
264 public Object invoke(Object proxy, Method method,
265 Object[] args) throws Throwable {
266 try {
267 if ((args == null || args.length == 0)
268 && "close".equals(method.getName())) {
269 RPC.stopProxy(cp);
270 return null;
271 } else {
272 Object res = method.invoke(cp, args);
273 if (res != null && args != null && args.length == 3
274 && "getBlockLocations".equals(method.getName())
275 && res instanceof LocatedBlocks
276 && args[0] instanceof String
277 && args[0] != null) {
278 lrb.reorderBlocks(conf, (LocatedBlocks) res, (String) args[0]);
279 }
280 return res;
281 }
282 } catch (InvocationTargetException ite) {
283
284
285 Throwable cause = ite.getCause();
286 if (cause == null){
287 throw new RuntimeException(
288 "Proxy invocation failed and getCause is null", ite);
289 }
290 if (cause instanceof UndeclaredThrowableException) {
291 Throwable causeCause = cause.getCause();
292 if (causeCause == null) {
293 throw new RuntimeException("UndeclaredThrowableException had null cause!");
294 }
295 cause = cause.getCause();
296 }
297 throw cause;
298 }
299 }
300 });
301 }
302
303
304
305
306 interface ReorderBlocks {
307
308
309
310
311
312
313
314 void reorderBlocks(Configuration conf, LocatedBlocks lbs, String src) throws IOException;
315 }
316
317
318
319
320
321
322 static class ReorderWALBlocks implements ReorderBlocks {
323 public void reorderBlocks(Configuration conf, LocatedBlocks lbs, String src)
324 throws IOException {
325
326 ServerName sn = HLogUtil.getServerNameFromHLogDirectoryName(conf, src);
327 if (sn == null) {
328
329 return;
330 }
331
332
333 String hostName = sn.getHostname();
334 if (LOG.isTraceEnabled()) {
335 LOG.trace(src +
336 " is an HLog file, so reordering blocks, last hostname will be:" + hostName);
337 }
338
339
340 for (LocatedBlock lb : lbs.getLocatedBlocks()) {
341 DatanodeInfo[] dnis = lb.getLocations();
342 if (dnis != null && dnis.length > 1) {
343 boolean found = false;
344 for (int i = 0; i < dnis.length - 1 && !found; i++) {
345 if (hostName.equals(dnis[i].getHostName())) {
346
347 DatanodeInfo toLast = dnis[i];
348 System.arraycopy(dnis, i + 1, dnis, i, dnis.length - i - 1);
349 dnis[dnis.length - 1] = toLast;
350 found = true;
351 }
352 }
353 }
354 }
355 }
356 }
357
358
359
360
361
362
363
364 static public FileSystem get(Configuration conf) throws IOException {
365 return new HFileSystem(conf, true);
366 }
367
368
369
370
371 static public FileSystem getLocalFs(Configuration conf) throws IOException {
372 return new HFileSystem(FileSystem.getLocal(conf));
373 }
374
375
376
377
378
379
380 @SuppressWarnings("deprecation")
381 public FSDataOutputStream createNonRecursive(Path f,
382 boolean overwrite,
383 int bufferSize, short replication, long blockSize,
384 Progressable progress) throws IOException {
385 return fs.createNonRecursive(f, overwrite, bufferSize, replication,
386 blockSize, progress);
387 }
388 }