1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.lang.reflect.Method;
25 import java.net.InetSocketAddress;
26 import java.net.URI;
27 import java.util.HashSet;
28 import java.util.Map;
29 import java.util.Set;
30
31 import com.google.common.collect.Sets;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hdfs.DistributedFileSystem;
38 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
39
40
41
42
43
44 public class FSHDFSUtils extends FSUtils{
45 private static final Log LOG = LogFactory.getLog(FSHDFSUtils.class);
46 private static Class dfsUtilClazz;
47 private static Method getNNAddressesMethod;
48
49
50
51
52
53
54 private static Set<InetSocketAddress> getNNAddresses(DistributedFileSystem fs,
55 Configuration conf) {
56 Set<InetSocketAddress> addresses = new HashSet<InetSocketAddress>();
57 String serviceName = fs.getCanonicalServiceName();
58
59 if (serviceName.startsWith("ha-hdfs")) {
60 try {
61 if (dfsUtilClazz == null) {
62 dfsUtilClazz = Class.forName("org.apache.hadoop.hdfs.DFSUtil");
63 }
64 if (getNNAddressesMethod == null) {
65 getNNAddressesMethod =
66 dfsUtilClazz.getMethod("getNNServiceRpcAddresses", Configuration.class);
67 }
68
69 Map<String, Map<String, InetSocketAddress>> addressMap =
70 (Map<String, Map<String, InetSocketAddress>>) getNNAddressesMethod
71 .invoke(null, conf);
72 for (Map.Entry<String, Map<String, InetSocketAddress>> entry : addressMap.entrySet()) {
73 Map<String, InetSocketAddress> nnMap = entry.getValue();
74 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
75 InetSocketAddress addr = e2.getValue();
76 addresses.add(addr);
77 }
78 }
79 } catch (Exception e) {
80 LOG.warn("DFSUtil.getNNServiceRpcAddresses failed. serviceName=" + serviceName, e);
81 }
82 } else {
83 URI uri = fs.getUri();
84 InetSocketAddress addr = new InetSocketAddress(uri.getHost(), uri.getPort());
85 addresses.add(addr);
86 }
87
88 return addresses;
89 }
90
91
92
93
94
95
96
97 public static boolean isSameHdfs(Configuration conf, FileSystem srcFs, FileSystem desFs) {
98
99
100 String srcServiceName = srcFs.getCanonicalServiceName();
101 String desServiceName = desFs.getCanonicalServiceName();
102
103 if (srcServiceName == null || desServiceName == null) {
104 return false;
105 }
106 if (srcServiceName.equals(desServiceName)) {
107 return true;
108 }
109 if (srcFs instanceof DistributedFileSystem && desFs instanceof DistributedFileSystem) {
110
111
112
113 Set<InetSocketAddress> srcAddrs = getNNAddresses((DistributedFileSystem) srcFs, conf);
114 Set<InetSocketAddress> desAddrs = getNNAddresses((DistributedFileSystem) desFs, conf);
115 if (Sets.intersection(srcAddrs, desAddrs).size() > 0) {
116 return true;
117 }
118 }
119
120 return false;
121 }
122
123
124
125
126 @Override
127 public void recoverFileLease(final FileSystem fs, final Path p,
128 Configuration conf)
129 throws IOException {
130 if (!isAppendSupported(conf)) {
131 LOG.warn("Running on HDFS without append enabled may result in data loss");
132 return;
133 }
134
135 if (!(fs instanceof DistributedFileSystem)) return;
136 recoverDFSFileLease((DistributedFileSystem)fs, p, conf);
137 }
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165 boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p,
166 final Configuration conf)
167 throws IOException {
168 LOG.info("Recovering lease on dfs file " + p);
169 long startWaiting = EnvironmentEdgeManager.currentTimeMillis();
170
171
172
173 long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 900000) + startWaiting;
174
175 long firstPause = conf.getInt("hbase.lease.recovery.first.pause", 3000);
176
177
178
179 long subsequentPause = conf.getInt("hbase.lease.recovery.dfs.timeout", 61 * 1000);
180
181 Method isFileClosedMeth = null;
182
183 boolean findIsFileClosedMeth = true;
184 boolean recovered = false;
185
186 for (int nbAttempt = 0; !recovered; nbAttempt++) {
187 recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
188 if (recovered) break;
189 if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)) break;
190 try {
191
192 if (nbAttempt == 0) {
193 Thread.sleep(firstPause);
194 } else {
195
196
197 long localStartWaiting = EnvironmentEdgeManager.currentTimeMillis();
198 while ((EnvironmentEdgeManager.currentTimeMillis() - localStartWaiting) <
199 subsequentPause) {
200 Thread.sleep(conf.getInt("hbase.lease.recovery.pause", 1000));
201 if (findIsFileClosedMeth) {
202 try {
203 isFileClosedMeth = dfs.getClass().getMethod("isFileClosed",
204 new Class[]{ Path.class });
205 } catch (NoSuchMethodException nsme) {
206 LOG.debug("isFileClosed not available");
207 } finally {
208 findIsFileClosedMeth = false;
209 }
210 }
211 if (isFileClosedMeth != null && isFileClosed(dfs, isFileClosedMeth, p)) {
212 recovered = true;
213 break;
214 }
215 }
216 }
217 } catch (InterruptedException ie) {
218 InterruptedIOException iioe = new InterruptedIOException();
219 iioe.initCause(ie);
220 throw iioe;
221 }
222 }
223 return recovered;
224 }
225
226 boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
227 final int nbAttempt, final Path p, final long startWaiting) {
228 if (recoveryTimeout < EnvironmentEdgeManager.currentTimeMillis()) {
229 LOG.warn("Cannot recoverLease after trying for " +
230 conf.getInt("hbase.lease.recovery.timeout", 900000) +
231 "ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; " +
232 getLogMessageDetail(nbAttempt, p, startWaiting));
233 return true;
234 }
235 return false;
236 }
237
238
239
240
241
242
243
244
245
246
247 boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p,
248 final long startWaiting)
249 throws FileNotFoundException {
250 boolean recovered = false;
251 try {
252 recovered = dfs.recoverLease(p);
253 LOG.info("recoverLease=" + recovered + ", " +
254 getLogMessageDetail(nbAttempt, p, startWaiting));
255 } catch (IOException e) {
256 if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) {
257
258 throw new FileNotFoundException("The given HLog wasn't found at " + p);
259 } else if (e instanceof FileNotFoundException) {
260 throw (FileNotFoundException)e;
261 }
262 LOG.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
263 }
264 return recovered;
265 }
266
267
268
269
270
271
272
273 private String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
274 return "attempt=" + nbAttempt + " on file=" + p + " after " +
275 (EnvironmentEdgeManager.currentTimeMillis() - startWaiting) + "ms";
276 }
277
278
279
280
281
282
283
284
285 private boolean isFileClosed(final DistributedFileSystem dfs, final Method m, final Path p) {
286 try {
287 return (Boolean) m.invoke(dfs, p);
288 } catch (SecurityException e) {
289 LOG.warn("No access", e);
290 } catch (Exception e) {
291 LOG.warn("Failed invocation for " + p.toString(), e);
292 }
293 return false;
294 }
295 }