1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver.wal;
22
23 import java.io.FilterInputStream;
24 import java.io.IOException;
25 import java.lang.reflect.Field;
26 import java.lang.reflect.Method;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FSDataInputStream;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.io.SequenceFile;
35
36 public class SequenceFileLogReader implements HLog.Reader {
37 private static final Log LOG = LogFactory.getLog(SequenceFileLogReader.class);
38
39
40
41
42
43
44
45
46
47
48
49
50
51 static class WALReader extends SequenceFile.Reader {
52
53 WALReader(final FileSystem fs, final Path p, final Configuration c)
54 throws IOException {
55 super(fs, p, c);
56 }
57
58 @Override
59 protected FSDataInputStream openFile(FileSystem fs, Path file,
60 int bufferSize, long length)
61 throws IOException {
62 return new WALReaderFSDataInputStream(super.openFile(fs, file,
63 bufferSize, length), length);
64 }
65
66
67
68
69
70
71 public boolean isWALCompressionEnabled() {
72 return SequenceFileLogWriter.isWALCompressionEnabled(this.getMetadata());
73 }
74
75
76
77
78 static class WALReaderFSDataInputStream extends FSDataInputStream {
79 private boolean firstGetPosInvocation = true;
80 private long length;
81
82 WALReaderFSDataInputStream(final FSDataInputStream is, final long l)
83 throws IOException {
84 super(is);
85 this.length = l;
86 }
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103 @Override
104 public long getPos() throws IOException {
105 if (this.firstGetPosInvocation) {
106 this.firstGetPosInvocation = false;
107 long adjust = 0;
108
109 try {
110 Field fIn = FilterInputStream.class.getDeclaredField("in");
111 fIn.setAccessible(true);
112 Object realIn = fIn.get(this.in);
113
114
115 if (realIn.getClass().getName().endsWith("DFSInputStream")) {
116 Method getFileLength = realIn.getClass().
117 getDeclaredMethod("getFileLength", new Class<?> []{});
118 getFileLength.setAccessible(true);
119 long realLength = ((Long)getFileLength.
120 invoke(realIn, new Object []{})).longValue();
121 assert(realLength >= this.length);
122 adjust = realLength - this.length;
123 } else {
124 LOG.info("Input stream class: " + realIn.getClass().getName() +
125 ", not adjusting length");
126 }
127 } catch(Exception e) {
128 SequenceFileLogReader.LOG.warn(
129 "Error while trying to get accurate file length. " +
130 "Truncation / data loss may occur if RegionServers die.", e);
131 }
132
133 return adjust + super.getPos();
134 }
135 return super.getPos();
136 }
137 }
138 }
139
140 Configuration conf;
141 WALReader reader;
142 FileSystem fs;
143
144
145 Path path;
146 int edit = 0;
147 long entryStart = 0;
148 boolean emptyCompressionContext = true;
149
150
151
152 protected CompressionContext compressionContext = null;
153
154 protected Class<? extends HLogKey> keyClass;
155
156
157
158
159 public SequenceFileLogReader() {
160 }
161
162
163
164
165
166
167
168 public SequenceFileLogReader(Class<? extends HLogKey> keyClass) {
169 this.keyClass = keyClass;
170 }
171
172 @Override
173 public void init(FileSystem fs, Path path, Configuration conf)
174 throws IOException {
175 this.conf = conf;
176 this.path = path;
177 reader = new WALReader(fs, path, conf);
178 this.fs = fs;
179
180
181 boolean compression = reader.isWALCompressionEnabled();
182 if (compression) {
183 try {
184 if (compressionContext == null) {
185 compressionContext = new CompressionContext(LRUDictionary.class);
186 } else {
187 compressionContext.clear();
188 }
189 } catch (Exception e) {
190 throw new IOException("Failed to initialize CompressionContext", e);
191 }
192 }
193 }
194
195 @Override
196 public void close() throws IOException {
197 try {
198 if (reader != null) {
199 this.reader.close();
200 this.reader = null;
201 }
202 } catch (IOException ioe) {
203 throw addFileInfoToException(ioe);
204 }
205 }
206
207 @Override
208 public HLog.Entry next() throws IOException {
209 return next(null);
210 }
211
212 @Override
213 public HLog.Entry next(HLog.Entry reuse) throws IOException {
214 this.entryStart = this.reader.getPosition();
215 HLog.Entry e = reuse;
216 if (e == null) {
217 HLogKey key;
218 if (keyClass == null) {
219 key = HLog.newKey(conf);
220 } else {
221 try {
222 key = keyClass.newInstance();
223 } catch (InstantiationException ie) {
224 throw new IOException(ie);
225 } catch (IllegalAccessException iae) {
226 throw new IOException(iae);
227 }
228 }
229
230 WALEdit val = new WALEdit();
231 e = new HLog.Entry(key, val);
232 }
233 boolean b = false;
234 try {
235 if (compressionContext != null) {
236 e.setCompressionContext(compressionContext);
237 }
238 b = this.reader.next(e.getKey(), e.getEdit());
239 } catch (IOException ioe) {
240 throw addFileInfoToException(ioe);
241 }
242 edit++;
243 if (compressionContext != null && emptyCompressionContext) {
244 emptyCompressionContext = false;
245 }
246 return b? e: null;
247 }
248
249 @Override
250 public void seek(long pos) throws IOException {
251 if (compressionContext != null && emptyCompressionContext) {
252 while (next() != null) {
253 if (getPosition() == pos) {
254 emptyCompressionContext = false;
255 break;
256 }
257 }
258 }
259 try {
260 reader.seek(pos);
261 } catch (IOException ioe) {
262 throw addFileInfoToException(ioe);
263 }
264 }
265
266 @Override
267 public long getPosition() throws IOException {
268 return reader != null ? reader.getPosition() : 0;
269 }
270
271 protected IOException addFileInfoToException(final IOException ioe)
272 throws IOException {
273 long pos = -1;
274 try {
275 pos = getPosition();
276 } catch (IOException e) {
277 LOG.warn("Failed getting position to add to throw", e);
278 }
279
280
281 long end = Long.MAX_VALUE;
282 try {
283 Field fEnd = SequenceFile.Reader.class.getDeclaredField("end");
284 fEnd.setAccessible(true);
285 end = fEnd.getLong(this.reader);
286 } catch(Exception e) {
287
288 String msg = (this.path == null? "": this.path.toString()) +
289 ", entryStart=" + entryStart + ", pos=" + pos +
290 ((end == Long.MAX_VALUE) ? "" : ", end=" + end) +
291 ", edit=" + this.edit;
292
293
294 try {
295 return (IOException) ioe.getClass()
296 .getConstructor(String.class)
297 .newInstance(msg)
298 .initCause(ioe);
299 } catch(Exception e) {
300
301 return ioe;
302 }
303
304 @Override
305 public void reset() throws IOException {
306
307
308 reader = new WALReader(fs, path, conf);
309 }
310 }