1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.security.Key;
22 import java.security.KeyException;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.KeyValue;
31 import org.apache.hadoop.hbase.fs.HFileSystem;
32 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
33 import org.apache.hadoop.hbase.io.crypto.Cipher;
34 import org.apache.hadoop.hbase.io.crypto.Encryption;
35 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
36 import org.apache.hadoop.hbase.security.EncryptionUtil;
37 import org.apache.hadoop.hbase.security.User;
38 import org.apache.hadoop.hbase.util.ByteBufferUtils;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.io.WritableUtils;
41
42
43
44
45 @InterfaceAudience.Private
46 public class HFileReaderV3 extends HFileReaderV2 {
47
48 private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
49
50 public static final int MAX_MINOR_VERSION = 0;
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70 public HFileReaderV3(final Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
71 final long size, final CacheConfig cacheConf, final HFileSystem hfs,
72 final Configuration conf) throws IOException {
73 super(path, trailer, fsdis, size, cacheConf, hfs, conf);
74 byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
75
76 if (tmp != null) {
77 hfileContext.setIncludesTags(true);
78 tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
79 if (tmp != null && Bytes.toBoolean(tmp)) {
80 hfileContext.setCompressTags(true);
81 }
82 }
83 }
84
85 @Override
86 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
87 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
88 trailer.expectMajorVersion(3);
89 HFileContextBuilder builder = new HFileContextBuilder()
90 .withIncludesMvcc(this.includesMemstoreTS)
91 .withHBaseCheckSum(true)
92 .withCompression(this.compressAlgo);
93
94
95 byte[] keyBytes = trailer.getEncryptionKey();
96 if (keyBytes != null) {
97 Encryption.Context cryptoContext = Encryption.newContext(conf);
98 Key key;
99 String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
100 User.getCurrent().getShortName());
101 try {
102
103 key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
104 } catch (KeyException e) {
105
106
107 if (LOG.isDebugEnabled()) {
108 LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
109 }
110 String alternateKeyName =
111 conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
112 if (alternateKeyName != null) {
113 try {
114 key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
115 } catch (KeyException ex) {
116 throw new IOException(ex);
117 }
118 } else {
119 throw new IOException(e);
120 }
121 }
122
123 Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
124 if (cipher == null) {
125 throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
126 }
127 cryptoContext.setCipher(cipher);
128 cryptoContext.setKey(key);
129 builder.withEncryptionContext(cryptoContext);
130 }
131
132 HFileContext context = builder.build();
133
134 if (LOG.isTraceEnabled()) {
135 LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
136 " initialized with cacheConf: " + cacheConf +
137 " comparator: " + comparator.getClass().getSimpleName() +
138 " fileContext: " + context);
139 }
140
141 return context;
142 }
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158 @Override
159 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
160 final boolean isCompaction) {
161 if (dataBlockEncoder.useEncodedScanner()) {
162 return new EncodedScannerV3(this, cacheBlocks, pread, isCompaction, this.hfileContext);
163 }
164 return new ScannerV3(this, cacheBlocks, pread, isCompaction);
165 }
166
167
168
169
170 protected static class ScannerV3 extends ScannerV2 {
171
172 private HFileReaderV3 reader;
173 private int currTagsLen;
174
175 public ScannerV3(HFileReaderV3 r, boolean cacheBlocks, final boolean pread,
176 final boolean isCompaction) {
177 super(r, cacheBlocks, pread, isCompaction);
178 this.reader = r;
179 }
180
181 @Override
182 protected int getCellBufSize() {
183 int kvBufSize = super.getCellBufSize();
184 if (reader.hfileContext.isIncludesTags()) {
185 kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
186 }
187 return kvBufSize;
188 }
189
190 @Override
191 public KeyValue getKeyValue() {
192 if (!isSeeked())
193 return null;
194 if (currTagsLen == 0) {
195
196 return formNoTagsKeyValue();
197 }
198 KeyValue ret = new KeyValue(blockBuffer.array(), blockBuffer.arrayOffset()
199 + blockBuffer.position(), getCellBufSize());
200 if (this.reader.shouldIncludeMemstoreTS()) {
201 ret.setMvccVersion(currMemstoreTS);
202 }
203 return ret;
204 }
205
206 protected void setNonSeekedState() {
207 super.setNonSeekedState();
208 currTagsLen = 0;
209 }
210
211 @Override
212 protected int getNextCellStartPosition() {
213 int nextKvPos = super.getNextCellStartPosition();
214 if (reader.hfileContext.isIncludesTags()) {
215 nextKvPos += Bytes.SIZEOF_SHORT + currTagsLen;
216 }
217 return nextKvPos;
218 }
219
220 protected void readKeyValueLen() {
221 blockBuffer.mark();
222 currKeyLen = blockBuffer.getInt();
223 currValueLen = blockBuffer.getInt();
224 if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
225 || currValueLen > blockBuffer.limit()) {
226 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
227 + currValueLen + ". Block offset: "
228 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
229 + blockBuffer.position() + " (without header).");
230 }
231 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
232 if (reader.hfileContext.isIncludesTags()) {
233
234 currTagsLen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
235 if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
236 throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
237 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
238 + blockBuffer.position() + " (without header).");
239 }
240 ByteBufferUtils.skip(blockBuffer, currTagsLen);
241 }
242 readMvccVersion();
243 blockBuffer.reset();
244 }
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265 protected int blockSeek(byte[] key, int offset, int length, boolean seekBefore) {
266 int klen, vlen, tlen = 0;
267 long memstoreTS = 0;
268 int memstoreTSLen = 0;
269 int lastKeyValueSize = -1;
270 do {
271 blockBuffer.mark();
272 klen = blockBuffer.getInt();
273 vlen = blockBuffer.getInt();
274 if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
275 || vlen > blockBuffer.limit()) {
276 throw new IllegalStateException("Invalid klen " + klen + " or vlen "
277 + vlen + ". Block offset: "
278 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
279 + blockBuffer.position() + " (without header).");
280 }
281 ByteBufferUtils.skip(blockBuffer, klen + vlen);
282 if (reader.hfileContext.isIncludesTags()) {
283
284 tlen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
285 if (tlen < 0 || tlen > blockBuffer.limit()) {
286 throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
287 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
288 + blockBuffer.position() + " (without header).");
289 }
290 ByteBufferUtils.skip(blockBuffer, tlen);
291 }
292 if (this.reader.shouldIncludeMemstoreTS()) {
293 if (this.reader.decodeMemstoreTS) {
294 memstoreTS = Bytes.readAsVLong(blockBuffer.array(), blockBuffer.arrayOffset()
295 + blockBuffer.position());
296 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
297 } else {
298 memstoreTS = 0;
299 memstoreTSLen = 1;
300 }
301 }
302 blockBuffer.reset();
303 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + (Bytes.SIZEOF_INT * 2);
304 int comp = reader.getComparator().compare(key, offset, length, blockBuffer.array(),
305 keyOffset, klen);
306
307 if (comp == 0) {
308 if (seekBefore) {
309 if (lastKeyValueSize < 0) {
310 throw new IllegalStateException("blockSeek with seekBefore "
311 + "at the first key of the block: key=" + Bytes.toStringBinary(key)
312 + ", blockOffset=" + block.getOffset() + ", onDiskSize="
313 + block.getOnDiskSizeWithHeader());
314 }
315 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
316 readKeyValueLen();
317 return 1;
318 }
319 currKeyLen = klen;
320 currValueLen = vlen;
321 currTagsLen = tlen;
322 if (this.reader.shouldIncludeMemstoreTS()) {
323 currMemstoreTS = memstoreTS;
324 currMemstoreTSLen = memstoreTSLen;
325 }
326 return 0;
327 } else if (comp < 0) {
328 if (lastKeyValueSize > 0)
329 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
330 readKeyValueLen();
331 if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
332 return HConstants.INDEX_KEY_MAGIC;
333 }
334 return 1;
335 }
336
337
338 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
339
340 if (reader.hfileContext.isIncludesTags()) {
341 lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
342 }
343 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
344 } while (blockBuffer.remaining() > 0);
345
346
347
348
349 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
350 readKeyValueLen();
351 return 1;
352 }
353
354 }
355
356
357
358
359 protected static class EncodedScannerV3 extends EncodedScannerV2 {
360 public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
361 boolean isCompaction, HFileContext context) {
362 super(reader, cacheBlocks, pread, isCompaction, context);
363 }
364 }
365
366 @Override
367 public int getMajorVersion() {
368 return 3;
369 }
370 }