1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.lang.reflect.Method;
25 import java.net.InetSocketAddress;
26 import java.net.URI;
27 import java.util.HashSet;
28 import java.util.Map;
29 import java.util.Set;
30
31 import com.google.common.collect.Sets;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.classification.InterfaceAudience;
35 import org.apache.hadoop.hbase.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hdfs.DistributedFileSystem;
40 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
41
42
43
44
45
46 @InterfaceAudience.Private
47 @InterfaceStability.Evolving
48 public class FSHDFSUtils extends FSUtils {
49 private static final Log LOG = LogFactory.getLog(FSHDFSUtils.class);
50 private static Class dfsUtilClazz;
51 private static Method getNNAddressesMethod;
52
53
54
55
56
57
58 private static Set<InetSocketAddress> getNNAddresses(DistributedFileSystem fs,
59 Configuration conf) {
60 Set<InetSocketAddress> addresses = new HashSet<InetSocketAddress>();
61 String serviceName = fs.getCanonicalServiceName();
62
63 if (serviceName.startsWith("ha-hdfs")) {
64 try {
65 if (dfsUtilClazz == null) {
66 dfsUtilClazz = Class.forName("org.apache.hadoop.hdfs.DFSUtil");
67 }
68 if (getNNAddressesMethod == null) {
69 getNNAddressesMethod =
70 dfsUtilClazz.getMethod("getNNServiceRpcAddresses", Configuration.class);
71 }
72
73 Map<String, Map<String, InetSocketAddress>> addressMap =
74 (Map<String, Map<String, InetSocketAddress>>) getNNAddressesMethod
75 .invoke(null, conf);
76 for (Map.Entry<String, Map<String, InetSocketAddress>> entry : addressMap.entrySet()) {
77 Map<String, InetSocketAddress> nnMap = entry.getValue();
78 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
79 InetSocketAddress addr = e2.getValue();
80 addresses.add(addr);
81 }
82 }
83 } catch (Exception e) {
84 LOG.warn("DFSUtil.getNNServiceRpcAddresses failed. serviceName=" + serviceName, e);
85 }
86 } else {
87 URI uri = fs.getUri();
88 int port = uri.getPort();
89 if (port < 0) {
90 int idx = serviceName.indexOf(':');
91 port = Integer.parseInt(serviceName.substring(idx+1));
92 }
93 InetSocketAddress addr = new InetSocketAddress(uri.getHost(), port);
94 addresses.add(addr);
95 }
96
97 return addresses;
98 }
99
100
101
102
103
104
105
106 public static boolean isSameHdfs(Configuration conf, FileSystem srcFs, FileSystem desFs) {
107
108
109 String srcServiceName = srcFs.getCanonicalServiceName();
110 String desServiceName = desFs.getCanonicalServiceName();
111
112 if (srcServiceName == null || desServiceName == null) {
113 return false;
114 }
115 if (srcServiceName.equals(desServiceName)) {
116 return true;
117 }
118 if (srcFs instanceof DistributedFileSystem && desFs instanceof DistributedFileSystem) {
119
120
121
122 Set<InetSocketAddress> srcAddrs = getNNAddresses((DistributedFileSystem) srcFs, conf);
123 Set<InetSocketAddress> desAddrs = getNNAddresses((DistributedFileSystem) desFs, conf);
124 if (Sets.intersection(srcAddrs, desAddrs).size() > 0) {
125 return true;
126 }
127 }
128
129 return false;
130 }
131
132
133
134
135 @Override
136 public void recoverFileLease(final FileSystem fs, final Path p,
137 Configuration conf, CancelableProgressable reporter)
138 throws IOException {
139
140 if (!(fs instanceof DistributedFileSystem)) return;
141 recoverDFSFileLease((DistributedFileSystem)fs, p, conf, reporter);
142 }
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170 boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p,
171 final Configuration conf, final CancelableProgressable reporter)
172 throws IOException {
173 LOG.info("Recovering lease on dfs file " + p);
174 long startWaiting = EnvironmentEdgeManager.currentTimeMillis();
175
176
177
178 long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 900000) + startWaiting;
179
180 long firstPause = conf.getInt("hbase.lease.recovery.first.pause", 4000);
181
182
183
184
185
186 long subsequentPauseBase = conf.getLong("hbase.lease.recovery.dfs.timeout", 61 * 1000);
187
188 Method isFileClosedMeth = null;
189
190 boolean findIsFileClosedMeth = true;
191 boolean recovered = false;
192
193 for (int nbAttempt = 0; !recovered; nbAttempt++) {
194 recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
195 if (recovered) break;
196 checkIfCancelled(reporter);
197 if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)) break;
198 try {
199
200 if (nbAttempt == 0) {
201 Thread.sleep(firstPause);
202 } else {
203
204
205 long localStartWaiting = EnvironmentEdgeManager.currentTimeMillis();
206 while ((EnvironmentEdgeManager.currentTimeMillis() - localStartWaiting) <
207 subsequentPauseBase * nbAttempt) {
208 Thread.sleep(conf.getInt("hbase.lease.recovery.pause", 1000));
209 if (findIsFileClosedMeth) {
210 try {
211 isFileClosedMeth = dfs.getClass().getMethod("isFileClosed",
212 new Class[]{ Path.class });
213 } catch (NoSuchMethodException nsme) {
214 LOG.debug("isFileClosed not available");
215 } finally {
216 findIsFileClosedMeth = false;
217 }
218 }
219 if (isFileClosedMeth != null && isFileClosed(dfs, isFileClosedMeth, p)) {
220 recovered = true;
221 break;
222 }
223 checkIfCancelled(reporter);
224 }
225 }
226 } catch (InterruptedException ie) {
227 InterruptedIOException iioe = new InterruptedIOException();
228 iioe.initCause(ie);
229 throw iioe;
230 }
231 }
232 return recovered;
233 }
234
235 boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
236 final int nbAttempt, final Path p, final long startWaiting) {
237 if (recoveryTimeout < EnvironmentEdgeManager.currentTimeMillis()) {
238 LOG.warn("Cannot recoverLease after trying for " +
239 conf.getInt("hbase.lease.recovery.timeout", 900000) +
240 "ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; " +
241 getLogMessageDetail(nbAttempt, p, startWaiting));
242 return true;
243 }
244 return false;
245 }
246
247
248
249
250
251
252
253
254
255
256 boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p,
257 final long startWaiting)
258 throws FileNotFoundException {
259 boolean recovered = false;
260 try {
261 recovered = dfs.recoverLease(p);
262 LOG.info("recoverLease=" + recovered + ", " +
263 getLogMessageDetail(nbAttempt, p, startWaiting));
264 } catch (IOException e) {
265 if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) {
266
267 throw new FileNotFoundException("The given HLog wasn't found at " + p);
268 } else if (e instanceof FileNotFoundException) {
269 throw (FileNotFoundException)e;
270 }
271 LOG.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
272 }
273 return recovered;
274 }
275
276
277
278
279
280
281
282 private String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
283 return "attempt=" + nbAttempt + " on file=" + p + " after " +
284 (EnvironmentEdgeManager.currentTimeMillis() - startWaiting) + "ms";
285 }
286
287
288
289
290
291
292
293
294 private boolean isFileClosed(final DistributedFileSystem dfs, final Method m, final Path p) {
295 try {
296 return (Boolean) m.invoke(dfs, p);
297 } catch (SecurityException e) {
298 LOG.warn("No access", e);
299 } catch (Exception e) {
300 LOG.warn("Failed invocation for " + p.toString(), e);
301 }
302 return false;
303 }
304
305 void checkIfCancelled(final CancelableProgressable reporter)
306 throws InterruptedIOException {
307 if (reporter == null) return;
308 if (!reporter.progress()) throw new InterruptedIOException("Operation cancelled");
309 }
310 }