1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.GZ;
22 import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.NONE;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.io.ByteArrayOutputStream;
28 import java.io.DataOutputStream;
29 import java.io.IOException;
30 import java.io.OutputStream;
31 import java.nio.ByteBuffer;
32 import java.util.ArrayList;
33 import java.util.Collection;
34 import java.util.List;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.fs.FSDataInputStream;
39 import org.apache.hadoop.fs.FSDataOutputStream;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.SmallTests;
44 import org.apache.hadoop.hbase.fs.HFileSystem;
45 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
46 import org.apache.hadoop.hbase.io.compress.Compression;
47 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
48 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
49 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
50 import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.hbase.util.ChecksumType;
53 import org.apache.hadoop.io.compress.Compressor;
54 import org.junit.Before;
55 import org.junit.Test;
56 import org.junit.experimental.categories.Category;
57 import org.junit.runner.RunWith;
58 import org.junit.runners.Parameterized;
59 import org.junit.runners.Parameterized.Parameters;
60
61 import com.google.common.base.Preconditions;
62
63
64
65
66
67 @Category(SmallTests.class)
68 @RunWith(Parameterized.class)
69 public class TestHFileBlockCompatibility {
70
71 private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
72
73 private static final Log LOG = LogFactory.getLog(TestHFileBlockCompatibility.class);
74
75 private static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
76 NONE, GZ };
77
78
79 private static int MINOR_VERSION = 0;
80
81 private static final HBaseTestingUtility TEST_UTIL =
82 new HBaseTestingUtility();
83 private HFileSystem fs;
84 private int uncompressedSizeV1;
85
86 private final boolean includesMemstoreTS;
87
88 public TestHFileBlockCompatibility(boolean includesMemstoreTS) {
89 this.includesMemstoreTS = includesMemstoreTS;
90 }
91
92 @Parameters
93 public static Collection<Object[]> parameters() {
94 return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
95 }
96
97 @Before
98 public void setUp() throws IOException {
99 fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
100 }
101
102 public byte[] createTestV1Block(Compression.Algorithm algo)
103 throws IOException {
104 Compressor compressor = algo.getCompressor();
105 ByteArrayOutputStream baos = new ByteArrayOutputStream();
106 OutputStream os = algo.createCompressionStream(baos, compressor, 0);
107 DataOutputStream dos = new DataOutputStream(os);
108 BlockType.META.write(dos);
109 TestHFileBlock.writeTestBlockContents(dos);
110 uncompressedSizeV1 = dos.size();
111 dos.flush();
112 algo.returnCompressor(compressor);
113 return baos.toByteArray();
114 }
115
116 private Writer createTestV2Block(Compression.Algorithm algo)
117 throws IOException {
118 final BlockType blockType = BlockType.DATA;
119 Writer hbw = new Writer(algo, null,
120 includesMemstoreTS);
121 DataOutputStream dos = hbw.startWriting(blockType);
122 TestHFileBlock.writeTestBlockContents(dos);
123
124 hbw.getHeaderAndData();
125 assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
126 hbw.releaseCompressor();
127 return hbw;
128 }
129
130 private String createTestBlockStr(Compression.Algorithm algo,
131 int correctLength) throws IOException {
132 Writer hbw = createTestV2Block(algo);
133 byte[] testV2Block = hbw.getHeaderAndData();
134 int osOffset = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM + 9;
135 if (testV2Block.length == correctLength) {
136
137
138
139 testV2Block[osOffset] = 3;
140 }
141 return Bytes.toStringBinary(testV2Block);
142 }
143
144 @Test
145 public void testNoCompression() throws IOException {
146 assertEquals(4000, createTestV2Block(NONE).getBlockForCaching().
147 getUncompressedSizeWithoutHeader());
148 }
149
150 @Test
151 public void testGzipCompression() throws IOException {
152 final String correctTestBlockStr =
153 "DATABLK*\\x00\\x00\\x00:\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
154 + "\\xFF\\xFF\\xFF\\xFF"
155
156 + "\\x1F\\x8B"
157 + "\\x08"
158 + "\\x00"
159 + "\\x00\\x00\\x00\\x00"
160 + "\\x00"
161
162
163 + "\\x03"
164 + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
165 + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
166 + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00";
167 final int correctGzipBlockLength = 82;
168
169 String returnedStr = createTestBlockStr(GZ, correctGzipBlockLength);
170 assertEquals(correctTestBlockStr, returnedStr);
171 }
172
173 @Test
174 public void testReaderV2() throws IOException {
175 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
176 for (boolean pread : new boolean[] { false, true }) {
177 LOG.info("testReaderV2: Compression algorithm: " + algo +
178 ", pread=" + pread);
179 Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
180 + algo);
181 FSDataOutputStream os = fs.create(path);
182 Writer hbw = new Writer(algo, null,
183 includesMemstoreTS);
184 long totalSize = 0;
185 for (int blockId = 0; blockId < 2; ++blockId) {
186 DataOutputStream dos = hbw.startWriting(BlockType.DATA);
187 for (int i = 0; i < 1234; ++i)
188 dos.writeInt(i);
189 hbw.writeHeaderAndData(os);
190 totalSize += hbw.getOnDiskSizeWithHeader();
191 }
192 os.close();
193
194 FSDataInputStream is = fs.open(path);
195 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
196 algo, totalSize, MINOR_VERSION, fs, path);
197 HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
198 is.close();
199
200 b.sanityCheck();
201 assertEquals(4936, b.getUncompressedSizeWithoutHeader());
202 assertEquals(algo == GZ ? 2173 : 4936,
203 b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
204 String blockStr = b.toString();
205
206 if (algo == GZ) {
207 is = fs.open(path);
208 hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
209 algo, totalSize, MINOR_VERSION, fs, path);
210 b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM +
211 b.totalChecksumBytes(), -1, pread);
212 assertEquals(blockStr, b.toString());
213 int wrongCompressedSize = 2172;
214 try {
215 b = hbr.readBlockData(0, wrongCompressedSize
216 + HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM, -1, pread);
217 fail("Exception expected");
218 } catch (IOException ex) {
219 String expectedPrefix = "On-disk size without header provided is "
220 + wrongCompressedSize + ", but block header contains "
221 + b.getOnDiskSizeWithoutHeader() + ".";
222 assertTrue("Invalid exception message: '" + ex.getMessage()
223 + "'.\nMessage is expected to start with: '" + expectedPrefix
224 + "'", ex.getMessage().startsWith(expectedPrefix));
225 }
226 is.close();
227 }
228 }
229 }
230 }
231
232
233
234
235
236 @Test
237 public void testDataBlockEncoding() throws IOException {
238 final int numBlocks = 5;
239 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
240 for (boolean pread : new boolean[] { false, true }) {
241 for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
242 LOG.info("testDataBlockEncoding algo " + algo +
243 " pread = " + pread +
244 " encoding " + encoding);
245 Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
246 + algo + "_" + encoding.toString());
247 FSDataOutputStream os = fs.create(path);
248 HFileDataBlockEncoder dataBlockEncoder =
249 new HFileDataBlockEncoderImpl(encoding);
250 TestHFileBlockCompatibility.Writer hbw =
251 new TestHFileBlockCompatibility.Writer(algo,
252 dataBlockEncoder, includesMemstoreTS);
253 long totalSize = 0;
254 final List<Integer> encodedSizes = new ArrayList<Integer>();
255 final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
256 for (int blockId = 0; blockId < numBlocks; ++blockId) {
257 DataOutputStream dos = hbw.startWriting(BlockType.DATA);
258 TestHFileBlock.writeEncodedBlock(algo, encoding, dos, encodedSizes,
259 encodedBlocks, blockId, includesMemstoreTS,
260 TestHFileBlockCompatibility.Writer.DUMMY_HEADER);
261
262 hbw.writeHeaderAndData(os);
263 totalSize += hbw.getOnDiskSizeWithHeader();
264 }
265 os.close();
266
267 FSDataInputStream is = fs.open(path);
268 HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
269 algo, totalSize, MINOR_VERSION, fs, path);
270 hbr.setDataBlockEncoder(dataBlockEncoder);
271 hbr.setIncludesMemstoreTS(includesMemstoreTS);
272
273 HFileBlock b;
274 int pos = 0;
275 for (int blockId = 0; blockId < numBlocks; ++blockId) {
276 b = hbr.readBlockData(pos, -1, -1, pread);
277 b.sanityCheck();
278 pos += b.getOnDiskSizeWithHeader();
279
280 assertEquals((int) encodedSizes.get(blockId),
281 b.getUncompressedSizeWithoutHeader());
282 ByteBuffer actualBuffer = b.getBufferWithoutHeader();
283 if (encoding != DataBlockEncoding.NONE) {
284
285 assertEquals(0, actualBuffer.get(0));
286 assertEquals(encoding.getId(), actualBuffer.get(1));
287 actualBuffer.position(2);
288 actualBuffer = actualBuffer.slice();
289 }
290
291 ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
292 expectedBuffer.rewind();
293
294
295 TestHFileBlock.assertBuffersEqual(expectedBuffer, actualBuffer,
296 algo, encoding, pread);
297 }
298 is.close();
299 }
300 }
301 }
302 }
303
304
305
306
307
308
309
310
311
312
313
314
315 public static final class Writer {
316
317
318 private static final int HEADER_SIZE = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
319 private static final boolean DONT_FILL_HEADER = HFileBlock.DONT_FILL_HEADER;
320 private static final byte[] DUMMY_HEADER =
321 HFileBlock.DUMMY_HEADER_NO_CHECKSUM;
322
323 private enum State {
324 INIT,
325 WRITING,
326 BLOCK_READY
327 };
328
329
330 private State state = State.INIT;
331
332
333 private final Compression.Algorithm compressAlgo;
334
335
336 private final HFileDataBlockEncoder dataBlockEncoder;
337
338 private HFileBlockEncodingContext dataBlockEncodingCtx;
339
340 private HFileBlockDefaultEncodingContext defaultBlockEncodingCtx;
341
342
343
344
345
346
347
348 private ByteArrayOutputStream baosInMemory;
349
350
351 private Compressor compressor;
352
353
354
355
356
357
358 private BlockType blockType;
359
360
361
362
363
364 private DataOutputStream userDataStream;
365
366
367
368
369
370 private byte[] onDiskBytesWithHeader;
371
372
373
374
375
376
377 private byte[] uncompressedBytesWithHeader;
378
379
380
381
382
383 private long startOffset;
384
385
386
387
388
389 private long[] prevOffsetByType;
390
391
392 private long prevOffset;
393
394
395 private boolean includesMemstoreTS;
396
397
398
399
400
401 public Writer(Compression.Algorithm compressionAlgorithm,
402 HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) {
403 compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
404 this.dataBlockEncoder = dataBlockEncoder != null
405 ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
406
407 defaultBlockEncodingCtx =
408 new HFileBlockDefaultEncodingContext(compressionAlgorithm,
409 null, DUMMY_HEADER);
410 dataBlockEncodingCtx =
411 this.dataBlockEncoder.newDataBlockEncodingContext(
412 compressionAlgorithm, DUMMY_HEADER);
413
414 baosInMemory = new ByteArrayOutputStream();
415
416 prevOffsetByType = new long[BlockType.values().length];
417 for (int i = 0; i < prevOffsetByType.length; ++i)
418 prevOffsetByType[i] = -1;
419
420 this.includesMemstoreTS = includesMemstoreTS;
421 }
422
423
424
425
426
427
428
429 public DataOutputStream startWriting(BlockType newBlockType)
430 throws IOException {
431 if (state == State.BLOCK_READY && startOffset != -1) {
432
433
434 prevOffsetByType[blockType.getId()] = startOffset;
435 }
436
437 startOffset = -1;
438 blockType = newBlockType;
439
440 baosInMemory.reset();
441 baosInMemory.write(DUMMY_HEADER);
442
443 state = State.WRITING;
444
445
446 userDataStream = new DataOutputStream(baosInMemory);
447 return userDataStream;
448 }
449
450
451
452
453
454
455
456
457 DataOutputStream getUserDataStream() {
458 expectState(State.WRITING);
459 return userDataStream;
460 }
461
462
463
464
465
466 private void ensureBlockReady() throws IOException {
467 Preconditions.checkState(state != State.INIT,
468 "Unexpected state: " + state);
469
470 if (state == State.BLOCK_READY)
471 return;
472
473
474 finishBlock();
475 }
476
477
478
479
480
481
482
483 private void finishBlock() throws IOException {
484 userDataStream.flush();
485
486 uncompressedBytesWithHeader = baosInMemory.toByteArray();
487 prevOffset = prevOffsetByType[blockType.getId()];
488
489
490
491
492 state = State.BLOCK_READY;
493 if (blockType == BlockType.DATA) {
494 encodeDataBlockForDisk();
495 } else {
496 defaultBlockEncodingCtx.compressAfterEncodingWithBlockType(
497 uncompressedBytesWithHeader, blockType);
498 onDiskBytesWithHeader =
499 defaultBlockEncodingCtx.getOnDiskBytesWithHeader();
500 }
501
502
503 putHeader(onDiskBytesWithHeader, 0,
504 onDiskBytesWithHeader.length,
505 uncompressedBytesWithHeader.length);
506
507 putHeader(uncompressedBytesWithHeader, 0,
508 onDiskBytesWithHeader.length,
509 uncompressedBytesWithHeader.length);
510 }
511
512
513
514
515
516 private void encodeDataBlockForDisk() throws IOException {
517
518 ByteBuffer rawKeyValues =
519 ByteBuffer.wrap(uncompressedBytesWithHeader, HEADER_SIZE,
520 uncompressedBytesWithHeader.length - HEADER_SIZE).slice();
521
522
523 dataBlockEncoder.beforeWriteToDisk(rawKeyValues,
524 includesMemstoreTS, dataBlockEncodingCtx, blockType);
525
526 uncompressedBytesWithHeader =
527 dataBlockEncodingCtx.getUncompressedBytesWithHeader();
528 onDiskBytesWithHeader =
529 dataBlockEncodingCtx.getOnDiskBytesWithHeader();
530 blockType = dataBlockEncodingCtx.getBlockType();
531 }
532
533
534
535
536
537
538
539 private void putHeader(byte[] dest, int offset, int onDiskSize,
540 int uncompressedSize) {
541 offset = blockType.put(dest, offset);
542 offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE);
543 offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE);
544 Bytes.putLong(dest, offset, prevOffset);
545 }
546
547
548
549
550
551
552
553
554
555 public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
556 long offset = out.getPos();
557 if (startOffset != -1 && offset != startOffset) {
558 throw new IOException("A " + blockType + " block written to a "
559 + "stream twice, first at offset " + startOffset + ", then at "
560 + offset);
561 }
562 startOffset = offset;
563
564 writeHeaderAndData((DataOutputStream) out);
565 }
566
567
568
569
570
571
572
573
574
575
576 private void writeHeaderAndData(DataOutputStream out) throws IOException {
577 ensureBlockReady();
578 out.write(onDiskBytesWithHeader);
579 }
580
581
582
583
584
585
586
587
588
589
590 public byte[] getHeaderAndData() throws IOException {
591 ensureBlockReady();
592 return onDiskBytesWithHeader;
593 }
594
595
596
597
598
599 public void releaseCompressor() {
600 if (compressor != null) {
601 compressAlgo.returnCompressor(compressor);
602 compressor = null;
603 }
604 }
605
606
607
608
609
610
611
612
613
614 public int getOnDiskSizeWithoutHeader() {
615 expectState(State.BLOCK_READY);
616 return onDiskBytesWithHeader.length - HEADER_SIZE;
617 }
618
619
620
621
622
623
624
625
626 public int getOnDiskSizeWithHeader() {
627 expectState(State.BLOCK_READY);
628 return onDiskBytesWithHeader.length;
629 }
630
631
632
633
634 public int getUncompressedSizeWithoutHeader() {
635 expectState(State.BLOCK_READY);
636 return uncompressedBytesWithHeader.length - HEADER_SIZE;
637 }
638
639
640
641
642 public int getUncompressedSizeWithHeader() {
643 expectState(State.BLOCK_READY);
644 return uncompressedBytesWithHeader.length;
645 }
646
647
648 public boolean isWriting() {
649 return state == State.WRITING;
650 }
651
652
653
654
655
656
657
658
659 public int blockSizeWritten() {
660 if (state != State.WRITING)
661 return 0;
662 return userDataStream.size();
663 }
664
665
666
667
668
669
670
671
672 private byte[] getUncompressedDataWithHeader() {
673 expectState(State.BLOCK_READY);
674
675 return uncompressedBytesWithHeader;
676 }
677
678 private void expectState(State expectedState) {
679 if (state != expectedState) {
680 throw new IllegalStateException("Expected state: " + expectedState +
681 ", actual state: " + state);
682 }
683 }
684
685
686
687
688
689
690
691 public ByteBuffer getUncompressedBufferWithHeader() {
692 byte[] b = getUncompressedDataWithHeader();
693 return ByteBuffer.wrap(b, 0, b.length);
694 }
695
696
697
698
699
700
701
702
703
704
705
706 public void writeBlock(BlockWritable bw, FSDataOutputStream out)
707 throws IOException {
708 bw.writeToBlock(startWriting(bw.getBlockType()));
709 writeHeaderAndData(out);
710 }
711
712
713
714
715 public HFileBlock getBlockForCaching() {
716 return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(),
717 getUncompressedSizeWithoutHeader(), prevOffset,
718 getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset,
719 includesMemstoreTS, MINOR_VERSION, 0, ChecksumType.NULL.getCode(),
720 getOnDiskSizeWithoutHeader());
721 }
722 }
723
724 }
725