1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.security.Key;
22 import java.security.KeyException;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.Cell;
30 import org.apache.hadoop.hbase.CellUtil;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.fs.HFileSystem;
34 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
35 import org.apache.hadoop.hbase.io.crypto.Cipher;
36 import org.apache.hadoop.hbase.io.crypto.Encryption;
37 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
38 import org.apache.hadoop.hbase.security.EncryptionUtil;
39 import org.apache.hadoop.hbase.security.User;
40 import org.apache.hadoop.hbase.util.ByteBufferUtils;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.io.WritableUtils;
43
44
45
46
47 @InterfaceAudience.Private
48 public class HFileReaderV3 extends HFileReaderV2 {
49
50 private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
51
52 public static final int MAX_MINOR_VERSION = 0;
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72 public HFileReaderV3(final Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
73 final long size, final CacheConfig cacheConf, final HFileSystem hfs,
74 final Configuration conf) throws IOException {
75 super(path, trailer, fsdis, size, cacheConf, hfs, conf);
76 byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
77
78 if (tmp != null) {
79 hfileContext.setIncludesTags(true);
80 tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
81 if (tmp != null && Bytes.toBoolean(tmp)) {
82 hfileContext.setCompressTags(true);
83 }
84 }
85 }
86
87 @Override
88 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
89 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
90 trailer.expectMajorVersion(3);
91 HFileContextBuilder builder = new HFileContextBuilder()
92 .withIncludesMvcc(this.includesMemstoreTS)
93 .withHBaseCheckSum(true)
94 .withCompression(this.compressAlgo);
95
96
97 byte[] keyBytes = trailer.getEncryptionKey();
98 if (keyBytes != null) {
99 Encryption.Context cryptoContext = Encryption.newContext(conf);
100 Key key;
101 String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
102 User.getCurrent().getShortName());
103 try {
104
105 key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
106 } catch (KeyException e) {
107
108
109 if (LOG.isDebugEnabled()) {
110 LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
111 }
112 String alternateKeyName =
113 conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
114 if (alternateKeyName != null) {
115 try {
116 key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
117 } catch (KeyException ex) {
118 throw new IOException(ex);
119 }
120 } else {
121 throw new IOException(e);
122 }
123 }
124
125 Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
126 if (cipher == null) {
127 throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
128 }
129 cryptoContext.setCipher(cipher);
130 cryptoContext.setKey(key);
131 builder.withEncryptionContext(cryptoContext);
132 }
133
134 HFileContext context = builder.build();
135
136 if (LOG.isTraceEnabled()) {
137 LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
138 " initialized with cacheConf: " + cacheConf +
139 " comparator: " + comparator.getClass().getSimpleName() +
140 " fileContext: " + context);
141 }
142
143 return context;
144 }
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160 @Override
161 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
162 final boolean isCompaction) {
163 if (dataBlockEncoder.useEncodedScanner()) {
164 return new EncodedScannerV3(this, cacheBlocks, pread, isCompaction, this.hfileContext);
165 }
166 return new ScannerV3(this, cacheBlocks, pread, isCompaction);
167 }
168
169
170
171
172 protected static class ScannerV3 extends ScannerV2 {
173
174 private HFileReaderV3 reader;
175 private int currTagsLen;
176
177 public ScannerV3(HFileReaderV3 r, boolean cacheBlocks, final boolean pread,
178 final boolean isCompaction) {
179 super(r, cacheBlocks, pread, isCompaction);
180 this.reader = r;
181 }
182
183 @Override
184 protected int getCellBufSize() {
185 int kvBufSize = super.getCellBufSize();
186 if (reader.hfileContext.isIncludesTags()) {
187 kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
188 }
189 return kvBufSize;
190 }
191
192 protected void setNonSeekedState() {
193 super.setNonSeekedState();
194 currTagsLen = 0;
195 }
196
197 @Override
198 protected int getNextCellStartPosition() {
199 int nextKvPos = super.getNextCellStartPosition();
200 if (reader.hfileContext.isIncludesTags()) {
201 nextKvPos += Bytes.SIZEOF_SHORT + currTagsLen;
202 }
203 return nextKvPos;
204 }
205
206 protected void readKeyValueLen() {
207 blockBuffer.mark();
208 currKeyLen = blockBuffer.getInt();
209 currValueLen = blockBuffer.getInt();
210 if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
211 || currValueLen > blockBuffer.limit()) {
212 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
213 + currValueLen + ". Block offset: "
214 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
215 + blockBuffer.position() + " (without header).");
216 }
217 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
218 if (reader.hfileContext.isIncludesTags()) {
219
220 currTagsLen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
221 if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
222 throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
223 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
224 + blockBuffer.position() + " (without header).");
225 }
226 ByteBufferUtils.skip(blockBuffer, currTagsLen);
227 }
228 readMvccVersion();
229 blockBuffer.reset();
230 }
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247 @Override
248 protected int blockSeek(Cell key, boolean seekBefore) {
249 int klen, vlen, tlen = 0;
250 long memstoreTS = 0;
251 int memstoreTSLen = 0;
252 int lastKeyValueSize = -1;
253 KeyValue.KeyOnlyKeyValue keyOnlyKv = new KeyValue.KeyOnlyKeyValue();
254 do {
255 blockBuffer.mark();
256 klen = blockBuffer.getInt();
257 vlen = blockBuffer.getInt();
258 if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
259 || vlen > blockBuffer.limit()) {
260 throw new IllegalStateException("Invalid klen " + klen + " or vlen "
261 + vlen + ". Block offset: "
262 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
263 + blockBuffer.position() + " (without header).");
264 }
265 ByteBufferUtils.skip(blockBuffer, klen + vlen);
266 if (reader.hfileContext.isIncludesTags()) {
267
268 tlen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
269 if (tlen < 0 || tlen > blockBuffer.limit()) {
270 throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
271 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
272 + blockBuffer.position() + " (without header).");
273 }
274 ByteBufferUtils.skip(blockBuffer, tlen);
275 }
276 if (this.reader.shouldIncludeMemstoreTS()) {
277 if (this.reader.decodeMemstoreTS) {
278 try {
279 memstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
280 + blockBuffer.position());
281 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
282 } catch (Exception e) {
283 throw new RuntimeException("Error reading memstore timestamp", e);
284 }
285 } else {
286 memstoreTS = 0;
287 memstoreTSLen = 1;
288 }
289 }
290 blockBuffer.reset();
291 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + (Bytes.SIZEOF_INT * 2);
292 keyOnlyKv.setKey(blockBuffer.array(), keyOffset, klen);
293 int comp = reader.getComparator().compareOnlyKeyPortion(key, keyOnlyKv);
294
295 if (comp == 0) {
296 if (seekBefore) {
297 if (lastKeyValueSize < 0) {
298 throw new IllegalStateException("blockSeek with seekBefore "
299 + "at the first key of the block: key="
300 + CellUtil.getCellKeyAsString(key)
301 + ", blockOffset=" + block.getOffset() + ", onDiskSize="
302 + block.getOnDiskSizeWithHeader());
303 }
304 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
305 readKeyValueLen();
306 return 1;
307 }
308 currKeyLen = klen;
309 currValueLen = vlen;
310 currTagsLen = tlen;
311 if (this.reader.shouldIncludeMemstoreTS()) {
312 currMemstoreTS = memstoreTS;
313 currMemstoreTSLen = memstoreTSLen;
314 }
315 return 0;
316 } else if (comp < 0) {
317 if (lastKeyValueSize > 0)
318 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
319 readKeyValueLen();
320 if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
321 return HConstants.INDEX_KEY_MAGIC;
322 }
323 return 1;
324 }
325
326
327 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
328
329 if (reader.hfileContext.isIncludesTags()) {
330 lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
331 }
332 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
333 } while (blockBuffer.remaining() > 0);
334
335
336
337
338 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
339 readKeyValueLen();
340 return 1;
341 }
342 }
343
344
345
346
347 protected static class EncodedScannerV3 extends EncodedScannerV2 {
348 public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
349 boolean isCompaction, HFileContext context) {
350 super(reader, cacheBlocks, pread, isCompaction, context);
351 }
352 }
353
354 @Override
355 public int getMajorVersion() {
356 return 3;
357 }
358 }