1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.lang.reflect.Method;
25 import java.net.InetSocketAddress;
26 import java.net.URI;
27 import java.util.HashSet;
28 import java.util.Map;
29 import java.util.Set;
30
31 import com.google.common.collect.Sets;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.classification.InterfaceAudience;
35 import org.apache.hadoop.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hdfs.DistributedFileSystem;
40 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
41
42
43
44
45
46 @InterfaceAudience.Private
47 @InterfaceStability.Evolving
48 public class FSHDFSUtils extends FSUtils {
49 private static final Log LOG = LogFactory.getLog(FSHDFSUtils.class);
50 private static Class dfsUtilClazz;
51 private static Method getNNAddressesMethod;
52
53
54
55
56
57
58 private static Set<InetSocketAddress> getNNAddresses(DistributedFileSystem fs,
59 Configuration conf) {
60 Set<InetSocketAddress> addresses = new HashSet<InetSocketAddress>();
61 String serviceName = fs.getCanonicalServiceName();
62
63 if (serviceName.startsWith("ha-hdfs")) {
64 try {
65 if (dfsUtilClazz == null) {
66 dfsUtilClazz = Class.forName("org.apache.hadoop.hdfs.DFSUtil");
67 }
68 if (getNNAddressesMethod == null) {
69 getNNAddressesMethod =
70 dfsUtilClazz.getMethod("getNNServiceRpcAddresses", Configuration.class);
71 }
72
73 Map<String, Map<String, InetSocketAddress>> addressMap =
74 (Map<String, Map<String, InetSocketAddress>>) getNNAddressesMethod
75 .invoke(null, conf);
76 for (Map.Entry<String, Map<String, InetSocketAddress>> entry : addressMap.entrySet()) {
77 Map<String, InetSocketAddress> nnMap = entry.getValue();
78 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
79 InetSocketAddress addr = e2.getValue();
80 addresses.add(addr);
81 }
82 }
83 } catch (Exception e) {
84 LOG.warn("DFSUtil.getNNServiceRpcAddresses failed. serviceName=" + serviceName, e);
85 }
86 } else {
87 URI uri = fs.getUri();
88 InetSocketAddress addr = new InetSocketAddress(uri.getHost(), uri.getPort());
89 addresses.add(addr);
90 }
91
92 return addresses;
93 }
94
95
96
97
98
99
100
101 public static boolean isSameHdfs(Configuration conf, FileSystem srcFs, FileSystem desFs) {
102
103
104 String srcServiceName = srcFs.getCanonicalServiceName();
105 String desServiceName = desFs.getCanonicalServiceName();
106
107 if (srcServiceName == null || desServiceName == null) {
108 return false;
109 }
110 if (srcServiceName.equals(desServiceName)) {
111 return true;
112 }
113 if (srcFs instanceof DistributedFileSystem && desFs instanceof DistributedFileSystem) {
114
115
116
117 Set<InetSocketAddress> srcAddrs = getNNAddresses((DistributedFileSystem) srcFs, conf);
118 Set<InetSocketAddress> desAddrs = getNNAddresses((DistributedFileSystem) desFs, conf);
119 if (Sets.intersection(srcAddrs, desAddrs).size() > 0) {
120 return true;
121 }
122 }
123
124 return false;
125 }
126
127
128
129
130 @Override
131 public void recoverFileLease(final FileSystem fs, final Path p,
132 Configuration conf, CancelableProgressable reporter)
133 throws IOException {
134
135 if (!(fs instanceof DistributedFileSystem)) return;
136 recoverDFSFileLease((DistributedFileSystem)fs, p, conf, reporter);
137 }
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165 boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p,
166 final Configuration conf, final CancelableProgressable reporter)
167 throws IOException {
168 LOG.info("Recovering lease on dfs file " + p);
169 long startWaiting = EnvironmentEdgeManager.currentTimeMillis();
170
171
172
173 long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 900000) + startWaiting;
174
175 long firstPause = conf.getInt("hbase.lease.recovery.first.pause", 4000);
176
177
178
179 long subsequentPause = conf.getInt("hbase.lease.recovery.dfs.timeout", 61 * 1000);
180
181 Method isFileClosedMeth = null;
182
183 boolean findIsFileClosedMeth = true;
184 boolean recovered = false;
185
186 for (int nbAttempt = 0; !recovered; nbAttempt++) {
187 recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
188 if (recovered) break;
189 checkIfCancelled(reporter);
190 if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)) break;
191 try {
192
193 if (nbAttempt == 0) {
194 Thread.sleep(firstPause);
195 } else {
196
197
198 long localStartWaiting = EnvironmentEdgeManager.currentTimeMillis();
199 while ((EnvironmentEdgeManager.currentTimeMillis() - localStartWaiting) <
200 subsequentPause) {
201 Thread.sleep(conf.getInt("hbase.lease.recovery.pause", 1000));
202 if (findIsFileClosedMeth) {
203 try {
204 isFileClosedMeth = dfs.getClass().getMethod("isFileClosed",
205 new Class[]{ Path.class });
206 } catch (NoSuchMethodException nsme) {
207 LOG.debug("isFileClosed not available");
208 } finally {
209 findIsFileClosedMeth = false;
210 }
211 }
212 if (isFileClosedMeth != null && isFileClosed(dfs, isFileClosedMeth, p)) {
213 recovered = true;
214 break;
215 }
216 checkIfCancelled(reporter);
217 }
218 }
219 } catch (InterruptedException ie) {
220 InterruptedIOException iioe = new InterruptedIOException();
221 iioe.initCause(ie);
222 throw iioe;
223 }
224 }
225 return recovered;
226 }
227
228 boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
229 final int nbAttempt, final Path p, final long startWaiting) {
230 if (recoveryTimeout < EnvironmentEdgeManager.currentTimeMillis()) {
231 LOG.warn("Cannot recoverLease after trying for " +
232 conf.getInt("hbase.lease.recovery.timeout", 900000) +
233 "ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; " +
234 getLogMessageDetail(nbAttempt, p, startWaiting));
235 return true;
236 }
237 return false;
238 }
239
240
241
242
243
244
245
246
247
248
249 boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p,
250 final long startWaiting)
251 throws FileNotFoundException {
252 boolean recovered = false;
253 try {
254 recovered = dfs.recoverLease(p);
255 LOG.info("recoverLease=" + recovered + ", " +
256 getLogMessageDetail(nbAttempt, p, startWaiting));
257 } catch (IOException e) {
258 if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) {
259
260 throw new FileNotFoundException("The given HLog wasn't found at " + p);
261 } else if (e instanceof FileNotFoundException) {
262 throw (FileNotFoundException)e;
263 }
264 LOG.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
265 }
266 return recovered;
267 }
268
269
270
271
272
273
274
275 private String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
276 return "attempt=" + nbAttempt + " on file=" + p + " after " +
277 (EnvironmentEdgeManager.currentTimeMillis() - startWaiting) + "ms";
278 }
279
280
281
282
283
284
285
286
287 private boolean isFileClosed(final DistributedFileSystem dfs, final Method m, final Path p) {
288 try {
289 return (Boolean) m.invoke(dfs, p);
290 } catch (SecurityException e) {
291 LOG.warn("No access", e);
292 } catch (Exception e) {
293 LOG.warn("Failed invocation for " + p.toString(), e);
294 }
295 return false;
296 }
297
298 void checkIfCancelled(final CancelableProgressable reporter)
299 throws InterruptedIOException {
300 if (reporter == null) return;
301 if (!reporter.progress()) throw new InterruptedIOException("Operation cancelled");
302 }
303 }