1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.lang.reflect.Method;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hdfs.DistributedFileSystem;
32 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
33
34
35
36
37
38 public class FSHDFSUtils extends FSUtils{
39 private static final Log LOG = LogFactory.getLog(FSHDFSUtils.class);
40
41
42
43
44 @Override
45 public void recoverFileLease(final FileSystem fs, final Path p,
46 Configuration conf)
47 throws IOException {
48 if (!isAppendSupported(conf)) {
49 LOG.warn("Running on HDFS without append enabled may result in data loss");
50 return;
51 }
52
53 if (!(fs instanceof DistributedFileSystem)) return;
54 recoverDFSFileLease((DistributedFileSystem)fs, p, conf);
55 }
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83 boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p,
84 final Configuration conf)
85 throws IOException {
86 LOG.info("Recovering lease on dfs file " + p);
87 long startWaiting = EnvironmentEdgeManager.currentTimeMillis();
88
89
90
91 long recoveryTimeout = conf.getInt("hbase.lease.recovery.timeout", 900000) + startWaiting;
92
93 long firstPause = conf.getInt("hbase.lease.recovery.first.pause", 3000);
94
95
96
97 long subsequentPause = conf.getInt("hbase.lease.recovery.dfs.timeout", 61 * 1000);
98
99 Method isFileClosedMeth = null;
100
101 boolean findIsFileClosedMeth = true;
102 boolean recovered = false;
103
104 for (int nbAttempt = 0; !recovered; nbAttempt++) {
105 recovered = recoverLease(dfs, nbAttempt, p, startWaiting);
106 if (recovered) break;
107 if (checkIfTimedout(conf, recoveryTimeout, nbAttempt, p, startWaiting)) break;
108 try {
109
110 if (nbAttempt == 0) {
111 Thread.sleep(firstPause);
112 } else {
113
114
115 long localStartWaiting = EnvironmentEdgeManager.currentTimeMillis();
116 while ((EnvironmentEdgeManager.currentTimeMillis() - localStartWaiting) <
117 subsequentPause) {
118 Thread.sleep(conf.getInt("hbase.lease.recovery.pause", 1000));
119 if (findIsFileClosedMeth) {
120 try {
121 isFileClosedMeth = dfs.getClass().getMethod("isFileClosed",
122 new Class[]{ Path.class });
123 } catch (NoSuchMethodException nsme) {
124 LOG.debug("isFileClosed not available");
125 } finally {
126 findIsFileClosedMeth = false;
127 }
128 }
129 if (isFileClosedMeth != null && isFileClosed(dfs, isFileClosedMeth, p)) {
130 recovered = true;
131 break;
132 }
133 }
134 }
135 } catch (InterruptedException ie) {
136 InterruptedIOException iioe = new InterruptedIOException();
137 iioe.initCause(ie);
138 throw iioe;
139 }
140 }
141 return recovered;
142 }
143
144 boolean checkIfTimedout(final Configuration conf, final long recoveryTimeout,
145 final int nbAttempt, final Path p, final long startWaiting) {
146 if (recoveryTimeout < EnvironmentEdgeManager.currentTimeMillis()) {
147 LOG.warn("Cannot recoverLease after trying for " +
148 conf.getInt("hbase.lease.recovery.timeout", 900000) +
149 "ms (hbase.lease.recovery.timeout); continuing, but may be DATALOSS!!!; " +
150 getLogMessageDetail(nbAttempt, p, startWaiting));
151 return true;
152 }
153 return false;
154 }
155
156
157
158
159
160
161
162
163
164
165 boolean recoverLease(final DistributedFileSystem dfs, final int nbAttempt, final Path p,
166 final long startWaiting)
167 throws FileNotFoundException {
168 boolean recovered = false;
169 try {
170 recovered = dfs.recoverLease(p);
171 LOG.info("recoverLease=" + recovered + ", " +
172 getLogMessageDetail(nbAttempt, p, startWaiting));
173 } catch (IOException e) {
174 if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) {
175
176 throw new FileNotFoundException("The given HLog wasn't found at " + p);
177 } else if (e instanceof FileNotFoundException) {
178 throw (FileNotFoundException)e;
179 }
180 LOG.warn(getLogMessageDetail(nbAttempt, p, startWaiting), e);
181 }
182 return recovered;
183 }
184
185
186
187
188
189
190
191 private String getLogMessageDetail(final int nbAttempt, final Path p, final long startWaiting) {
192 return "attempt=" + nbAttempt + " on file=" + p + " after " +
193 (EnvironmentEdgeManager.currentTimeMillis() - startWaiting) + "ms";
194 }
195
196
197
198
199
200
201
202
203 private boolean isFileClosed(final DistributedFileSystem dfs, final Method m, final Path p) {
204 try {
205 return (Boolean) m.invoke(dfs, p);
206 } catch (SecurityException e) {
207 LOG.warn("No access", e);
208 } catch (Exception e) {
209 LOG.warn("Failed invocation for " + p.toString(), e);
210 }
211 return false;
212 }
213 }