1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.junit.Assert.*;
23
24 import java.io.ByteArrayOutputStream;
25 import java.io.DataOutputStream;
26 import java.io.IOException;
27 import java.io.OutputStream;
28 import java.nio.ByteBuffer;
29 import java.util.ArrayList;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.HashMap;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.Random;
36 import java.util.concurrent.Callable;
37 import java.util.concurrent.Executor;
38 import java.util.concurrent.ExecutorCompletionService;
39 import java.util.concurrent.Executors;
40 import java.util.concurrent.Future;
41
42 import org.apache.commons.logging.Log;
43 import org.apache.commons.logging.LogFactory;
44 import org.apache.hadoop.fs.FSDataInputStream;
45 import org.apache.hadoop.fs.FSDataOutputStream;
46 import org.apache.hadoop.fs.FileSystem;
47 import org.apache.hadoop.fs.Path;
48 import org.apache.hadoop.hbase.HBaseTestingUtility;
49 import org.apache.hadoop.hbase.MediumTests;
50 import org.apache.hadoop.hbase.KeyValue;
51 import org.apache.hadoop.hbase.fs.HFileSystem;
52 import org.apache.hadoop.hbase.io.DoubleOutputStream;
53 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.ChecksumType;
56 import org.apache.hadoop.hbase.util.ClassSize;
57 import org.apache.hadoop.io.WritableUtils;
58 import org.apache.hadoop.io.compress.CompressionOutputStream;
59 import org.apache.hadoop.io.compress.Compressor;
60
61 import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
62 import org.junit.Before;
63 import org.junit.Test;
64 import org.junit.experimental.categories.Category;
65 import org.junit.runner.RunWith;
66 import org.junit.runners.Parameterized;
67 import org.junit.runners.Parameterized.Parameters;
68
69 @Category(MediumTests.class)
70 @RunWith(Parameterized.class)
71 public class TestHFileBlock {
72
73 private static final boolean detailedLogging = false;
74 private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
75
76 private static final Log LOG = LogFactory.getLog(TestHFileBlock.class);
77
78 static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
79 NONE, GZ };
80
81 private static final int NUM_TEST_BLOCKS = 1000;
82 private static final int NUM_READER_THREADS = 26;
83
84
85 private static int NUM_KEYVALUES = 50;
86 private static int FIELD_LENGTH = 10;
87 private static float CHANCE_TO_REPEAT = 0.6f;
88
89 private static final HBaseTestingUtility TEST_UTIL =
90 new HBaseTestingUtility();
91 private FileSystem fs;
92 private int uncompressedSizeV1;
93
94 private final boolean includesMemstoreTS;
95
96 public TestHFileBlock(boolean includesMemstoreTS) {
97 this.includesMemstoreTS = includesMemstoreTS;
98 }
99
100 @Parameters
101 public static Collection<Object[]> parameters() {
102 return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
103 }
104
105 @Before
106 public void setUp() throws IOException {
107 fs = HFileSystem.get(TEST_UTIL.getConfiguration());
108 }
109
110 static void writeTestBlockContents(DataOutputStream dos) throws IOException {
111
112 for (int i = 0; i < 1000; ++i)
113 dos.writeInt(i / 100);
114 }
115
116 static int writeTestKeyValues(OutputStream dos, int seed, boolean includesMemstoreTS)
117 throws IOException {
118 List<KeyValue> keyValues = new ArrayList<KeyValue>();
119 Random randomizer = new Random(42l + seed);
120
121
122 for (int i = 0; i < NUM_KEYVALUES; ++i) {
123 byte[] row;
124 long timestamp;
125 byte[] family;
126 byte[] qualifier;
127 byte[] value;
128
129
130 if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
131 row = keyValues.get(randomizer.nextInt(keyValues.size())).getRow();
132 } else {
133 row = new byte[FIELD_LENGTH];
134 randomizer.nextBytes(row);
135 }
136 if (0 == i) {
137 family = new byte[FIELD_LENGTH];
138 randomizer.nextBytes(family);
139 } else {
140 family = keyValues.get(0).getFamily();
141 }
142 if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
143 qualifier = keyValues.get(
144 randomizer.nextInt(keyValues.size())).getQualifier();
145 } else {
146 qualifier = new byte[FIELD_LENGTH];
147 randomizer.nextBytes(qualifier);
148 }
149 if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
150 value = keyValues.get(randomizer.nextInt(keyValues.size())).getValue();
151 } else {
152 value = new byte[FIELD_LENGTH];
153 randomizer.nextBytes(value);
154 }
155 if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
156 timestamp = keyValues.get(
157 randomizer.nextInt(keyValues.size())).getTimestamp();
158 } else {
159 timestamp = randomizer.nextLong();
160 }
161
162 keyValues.add(new KeyValue(row, family, qualifier, timestamp, value));
163 }
164
165
166 int totalSize = 0;
167 Collections.sort(keyValues, KeyValue.COMPARATOR);
168 DataOutputStream dataOutputStream = new DataOutputStream(dos);
169 for (KeyValue kv : keyValues) {
170 totalSize += kv.getLength();
171 dataOutputStream.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
172 if (includesMemstoreTS) {
173 long memstoreTS = randomizer.nextLong();
174 WritableUtils.writeVLong(dataOutputStream, memstoreTS);
175 totalSize += WritableUtils.getVIntSize(memstoreTS);
176 }
177 }
178
179 return totalSize;
180 }
181
182 public byte[] createTestV1Block(Compression.Algorithm algo)
183 throws IOException {
184 Compressor compressor = algo.getCompressor();
185 ByteArrayOutputStream baos = new ByteArrayOutputStream();
186 OutputStream os = algo.createCompressionStream(baos, compressor, 0);
187 DataOutputStream dos = new DataOutputStream(os);
188 BlockType.META.write(dos);
189 writeTestBlockContents(dos);
190 uncompressedSizeV1 = dos.size();
191 dos.flush();
192 algo.returnCompressor(compressor);
193 return baos.toByteArray();
194 }
195
196 static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
197 boolean includesMemstoreTS) throws IOException {
198 final BlockType blockType = BlockType.DATA;
199 HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
200 includesMemstoreTS, HFileReaderV2.MAX_MINOR_VERSION,
201 HFile.DEFAULT_CHECKSUM_TYPE,
202 HFile.DEFAULT_BYTES_PER_CHECKSUM);
203 DataOutputStream dos = hbw.startWriting(blockType);
204 writeTestBlockContents(dos);
205 dos.flush();
206 byte[] headerAndData = hbw.getHeaderAndDataForTest();
207 assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
208 hbw.releaseCompressor();
209 return hbw;
210 }
211
212 public String createTestBlockStr(Compression.Algorithm algo,
213 int correctLength) throws IOException {
214 HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS);
215 byte[] testV2Block = hbw.getHeaderAndDataForTest();
216 int osOffset = HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + 9;
217 if (testV2Block.length == correctLength) {
218
219
220
221
222
223 testV2Block[osOffset] = 3;
224 }
225 return Bytes.toStringBinary(testV2Block);
226 }
227
228 @Test
229 public void testNoCompression() throws IOException {
230 assertEquals(4000, createTestV2Block(NONE, includesMemstoreTS).
231 getBlockForCaching().getUncompressedSizeWithoutHeader());
232 }
233
234 @Test
235 public void testGzipCompression() throws IOException {
236 final String correctTestBlockStr =
237 "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
238 + "\\xFF\\xFF\\xFF\\xFF"
239 + "\\x01\\x00\\x00@\\x00\\x00\\x00\\x00["
240
241 + "\\x1F\\x8B"
242 + "\\x08"
243 + "\\x00"
244 + "\\x00\\x00\\x00\\x00"
245 + "\\x00"
246
247
248
249
250 + "\\x03"
251 + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
252 + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
253 + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00"
254 + "\\x00\\x00\\x00\\x00";
255 final int correctGzipBlockLength = 95;
256 final String testBlockStr = createTestBlockStr(GZ, correctGzipBlockLength);
257
258
259 assertEquals(correctTestBlockStr.substring(0, correctGzipBlockLength - 4),
260 testBlockStr.substring(0, correctGzipBlockLength - 4));
261 }
262
263 @Test
264 public void testReaderV1() throws IOException {
265 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
266 for (boolean pread : new boolean[] { false, true }) {
267 byte[] block = createTestV1Block(algo);
268 Path path = new Path(TEST_UTIL.getDataTestDir(),
269 "blocks_v1_"+ algo);
270 LOG.info("Creating temporary file at " + path);
271 FSDataOutputStream os = fs.create(path);
272 int totalSize = 0;
273 int numBlocks = 50;
274 for (int i = 0; i < numBlocks; ++i) {
275 os.write(block);
276 totalSize += block.length;
277 }
278 os.close();
279
280 FSDataInputStream is = fs.open(path);
281 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV1(is, algo,
282 totalSize);
283 HFileBlock b;
284 int numBlocksRead = 0;
285 long pos = 0;
286 while (pos < totalSize) {
287 b = hbr.readBlockData(pos, block.length, uncompressedSizeV1, pread);
288 b.sanityCheck();
289 pos += block.length;
290 numBlocksRead++;
291 }
292 assertEquals(numBlocks, numBlocksRead);
293 is.close();
294 }
295 }
296 }
297
298 @Test
299 public void testReaderV2() throws IOException {
300 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
301 for (boolean pread : new boolean[] { false, true }) {
302 LOG.info("testReaderV2: Compression algorithm: " + algo +
303 ", pread=" + pread);
304 Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
305 + algo);
306 FSDataOutputStream os = fs.create(path);
307 HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
308 includesMemstoreTS,
309 HFileReaderV2.MAX_MINOR_VERSION,
310 HFile.DEFAULT_CHECKSUM_TYPE,
311 HFile.DEFAULT_BYTES_PER_CHECKSUM);
312 long totalSize = 0;
313 for (int blockId = 0; blockId < 2; ++blockId) {
314 DataOutputStream dos = hbw.startWriting(BlockType.DATA);
315 for (int i = 0; i < 1234; ++i)
316 dos.writeInt(i);
317 hbw.writeHeaderAndData(os);
318 totalSize += hbw.getOnDiskSizeWithHeader();
319 }
320 os.close();
321
322 FSDataInputStream is = fs.open(path);
323 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo,
324 totalSize);
325 HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
326 is.close();
327 assertEquals(0, HFile.getChecksumFailuresCount());
328
329 b.sanityCheck();
330 assertEquals(4936, b.getUncompressedSizeWithoutHeader());
331 assertEquals(algo == GZ ? 2173 : 4936,
332 b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
333 String blockStr = b.toString();
334
335 if (algo == GZ) {
336 is = fs.open(path);
337 hbr = new HFileBlock.FSReaderV2(is, algo, totalSize);
338 b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS +
339 b.totalChecksumBytes(), -1, pread);
340 assertEquals(blockStr, b.toString());
341 int wrongCompressedSize = 2172;
342 try {
343 b = hbr.readBlockData(0, wrongCompressedSize
344 + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS, -1, pread);
345 fail("Exception expected");
346 } catch (IOException ex) {
347 String expectedPrefix = "On-disk size without header provided is "
348 + wrongCompressedSize + ", but block header contains "
349 + b.getOnDiskSizeWithoutHeader() + ".";
350 assertTrue("Invalid exception message: '" + ex.getMessage()
351 + "'.\nMessage is expected to start with: '" + expectedPrefix
352 + "'", ex.getMessage().startsWith(expectedPrefix));
353 }
354 is.close();
355 }
356 }
357 }
358 }
359
360
361
362
363
364 @Test
365 public void testDataBlockEncoding() throws IOException {
366 final int numBlocks = 5;
367 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
368 for (boolean pread : new boolean[] { false, true }) {
369 for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
370 Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
371 + algo + "_" + encoding.toString());
372 FSDataOutputStream os = fs.create(path);
373 HFileDataBlockEncoder dataBlockEncoder =
374 new HFileDataBlockEncoderImpl(encoding);
375 HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder,
376 includesMemstoreTS,
377 HFileReaderV2.MAX_MINOR_VERSION,
378 HFile.DEFAULT_CHECKSUM_TYPE,
379 HFile.DEFAULT_BYTES_PER_CHECKSUM);
380 long totalSize = 0;
381 final List<Integer> encodedSizes = new ArrayList<Integer>();
382 final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
383 for (int blockId = 0; blockId < numBlocks; ++blockId) {
384 DataOutputStream dos = hbw.startWriting(BlockType.DATA);
385 writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks,
386 blockId, includesMemstoreTS);
387
388 hbw.writeHeaderAndData(os);
389 totalSize += hbw.getOnDiskSizeWithHeader();
390 }
391 os.close();
392
393 FSDataInputStream is = fs.open(path);
394 HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, algo,
395 totalSize);
396 hbr.setDataBlockEncoder(dataBlockEncoder);
397 hbr.setIncludesMemstoreTS(includesMemstoreTS);
398
399 HFileBlock b;
400 int pos = 0;
401 for (int blockId = 0; blockId < numBlocks; ++blockId) {
402 b = hbr.readBlockData(pos, -1, -1, pread);
403 assertEquals(0, HFile.getChecksumFailuresCount());
404 b.sanityCheck();
405 pos += b.getOnDiskSizeWithHeader();
406
407 assertEquals((int) encodedSizes.get(blockId),
408 b.getUncompressedSizeWithoutHeader());
409 ByteBuffer actualBuffer = b.getBufferWithoutHeader();
410 if (encoding != DataBlockEncoding.NONE) {
411
412 assertEquals(0, actualBuffer.get(0));
413 assertEquals(encoding.getId(), actualBuffer.get(1));
414 actualBuffer.position(2);
415 actualBuffer = actualBuffer.slice();
416 }
417
418 ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
419 expectedBuffer.rewind();
420
421
422 assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding,
423 pread);
424 }
425 is.close();
426 }
427 }
428 }
429 }
430
431 static void writeEncodedBlock(DataBlockEncoding encoding,
432 DataOutputStream dos, final List<Integer> encodedSizes,
433 final List<ByteBuffer> encodedBlocks, int blockId,
434 boolean includesMemstoreTS) throws IOException {
435 ByteArrayOutputStream baos = new ByteArrayOutputStream();
436 DoubleOutputStream doubleOutputStream =
437 new DoubleOutputStream(dos, baos);
438
439 final int rawBlockSize = writeTestKeyValues(doubleOutputStream,
440 blockId, includesMemstoreTS);
441
442 ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
443 rawBuf.rewind();
444
445 final int encodedSize;
446 final ByteBuffer encodedBuf;
447 if (encoding == DataBlockEncoding.NONE) {
448 encodedSize = rawBlockSize;
449 encodedBuf = rawBuf;
450 } else {
451 ByteArrayOutputStream encodedOut = new ByteArrayOutputStream();
452 encoding.getEncoder().compressKeyValues(
453 new DataOutputStream(encodedOut),
454 rawBuf.duplicate(), includesMemstoreTS);
455
456
457 encodedSize = encodedOut.size() + DataBlockEncoding.ID_SIZE;
458 encodedBuf = ByteBuffer.wrap(encodedOut.toByteArray());
459 }
460 encodedSizes.add(encodedSize);
461 encodedBlocks.add(encodedBuf);
462 }
463
464 static void assertBuffersEqual(ByteBuffer expectedBuffer,
465 ByteBuffer actualBuffer, Compression.Algorithm compression,
466 DataBlockEncoding encoding, boolean pread) {
467 if (!actualBuffer.equals(expectedBuffer)) {
468 int prefix = 0;
469 int minLimit = Math.min(expectedBuffer.limit(), actualBuffer.limit());
470 while (prefix < minLimit &&
471 expectedBuffer.get(prefix) == actualBuffer.get(prefix)) {
472 prefix++;
473 }
474
475 fail(String.format(
476 "Content mismath for compression %s, encoding %s, " +
477 "pread %s, commonPrefix %d, expected %s, got %s",
478 compression, encoding, pread, prefix,
479 nextBytesToStr(expectedBuffer, prefix),
480 nextBytesToStr(actualBuffer, prefix)));
481 }
482 }
483
484
485
486
487
488 private static String nextBytesToStr(ByteBuffer buf, int pos) {
489 int maxBytes = buf.limit() - pos;
490 int numBytes = Math.min(16, maxBytes);
491 return Bytes.toStringBinary(buf.array(), buf.arrayOffset() + pos,
492 numBytes) + (numBytes < maxBytes ? "..." : "");
493 }
494
495 @Test
496 public void testPreviousOffset() throws IOException {
497 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
498 for (boolean pread : BOOLEAN_VALUES) {
499 for (boolean cacheOnWrite : BOOLEAN_VALUES) {
500 Random rand = defaultRandom();
501 LOG.info("testPreviousOffset:Compression algorithm: " + algo +
502 ", pread=" + pread +
503 ", cacheOnWrite=" + cacheOnWrite);
504 Path path = new Path(TEST_UTIL.getDataTestDir(), "prev_offset");
505 List<Long> expectedOffsets = new ArrayList<Long>();
506 List<Long> expectedPrevOffsets = new ArrayList<Long>();
507 List<BlockType> expectedTypes = new ArrayList<BlockType>();
508 List<ByteBuffer> expectedContents = cacheOnWrite
509 ? new ArrayList<ByteBuffer>() : null;
510 long totalSize = writeBlocks(rand, algo, path, expectedOffsets,
511 expectedPrevOffsets, expectedTypes, expectedContents);
512
513 FSDataInputStream is = fs.open(path);
514 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo,
515 totalSize);
516 long curOffset = 0;
517 for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
518 if (!pread) {
519 assertEquals(is.getPos(), curOffset + (i == 0 ? 0 :
520 HFileBlock.HEADER_SIZE_WITH_CHECKSUMS));
521 }
522
523 assertEquals(expectedOffsets.get(i).longValue(), curOffset);
524 if (detailedLogging) {
525 LOG.info("Reading block #" + i + " at offset " + curOffset);
526 }
527 HFileBlock b = hbr.readBlockData(curOffset, -1, -1, pread);
528 if (detailedLogging) {
529 LOG.info("Block #" + i + ": " + b);
530 }
531 assertEquals("Invalid block #" + i + "'s type:",
532 expectedTypes.get(i), b.getBlockType());
533 assertEquals("Invalid previous block offset for block " + i
534 + " of " + "type " + b.getBlockType() + ":",
535 (long) expectedPrevOffsets.get(i), b.getPrevBlockOffset());
536 b.sanityCheck();
537 assertEquals(curOffset, b.getOffset());
538
539
540
541 HFileBlock b2 = hbr.readBlockData(curOffset,
542 b.getOnDiskSizeWithHeader(), -1, pread);
543 b2.sanityCheck();
544
545 assertEquals(b.getBlockType(), b2.getBlockType());
546 assertEquals(b.getOnDiskSizeWithoutHeader(),
547 b2.getOnDiskSizeWithoutHeader());
548 assertEquals(b.getOnDiskSizeWithHeader(),
549 b2.getOnDiskSizeWithHeader());
550 assertEquals(b.getUncompressedSizeWithoutHeader(),
551 b2.getUncompressedSizeWithoutHeader());
552 assertEquals(b.getPrevBlockOffset(), b2.getPrevBlockOffset());
553 assertEquals(curOffset, b2.getOffset());
554 assertEquals(b.getBytesPerChecksum(), b2.getBytesPerChecksum());
555 assertEquals(b.getOnDiskDataSizeWithHeader(),
556 b2.getOnDiskDataSizeWithHeader());
557 assertEquals(0, HFile.getChecksumFailuresCount());
558
559 curOffset += b.getOnDiskSizeWithHeader();
560
561 if (cacheOnWrite) {
562
563
564
565
566 ByteBuffer bufRead = b.getBufferWithHeader();
567 ByteBuffer bufExpected = expectedContents.get(i);
568 boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
569 bufRead.arrayOffset(),
570 bufRead.limit() - b.totalChecksumBytes(),
571 bufExpected.array(), bufExpected.arrayOffset(),
572 bufExpected.limit()) == 0;
573 String wrongBytesMsg = "";
574
575 if (!bytesAreCorrect) {
576
577
578 wrongBytesMsg = "Expected bytes in block #" + i + " (algo="
579 + algo + ", pread=" + pread
580 + ", cacheOnWrite=" + cacheOnWrite + "):\n";
581 wrongBytesMsg += Bytes.toStringBinary(bufExpected.array(),
582 bufExpected.arrayOffset(), Math.min(32,
583 bufExpected.limit()))
584 + ", actual:\n"
585 + Bytes.toStringBinary(bufRead.array(),
586 bufRead.arrayOffset(), Math.min(32, bufRead.limit()));
587 if (detailedLogging) {
588 LOG.warn("expected header" +
589 HFileBlock.toStringHeader(bufExpected) +
590 "\nfound header" +
591 HFileBlock.toStringHeader(bufRead));
592 LOG.warn("bufread offset " + bufRead.arrayOffset() +
593 " limit " + bufRead.limit() +
594 " expected offset " + bufExpected.arrayOffset() +
595 " limit " + bufExpected.limit());
596 LOG.warn(wrongBytesMsg);
597 }
598 }
599 assertTrue(wrongBytesMsg, bytesAreCorrect);
600 }
601 }
602
603 assertEquals(curOffset, fs.getFileStatus(path).getLen());
604 is.close();
605 }
606 }
607 }
608 }
609
610 private Random defaultRandom() {
611 return new Random(189237);
612 }
613
614 private class BlockReaderThread implements Callable<Boolean> {
615 private final String clientId;
616 private final HFileBlock.FSReader hbr;
617 private final List<Long> offsets;
618 private final List<BlockType> types;
619 private final long fileSize;
620
621 public BlockReaderThread(String clientId,
622 HFileBlock.FSReader hbr, List<Long> offsets, List<BlockType> types,
623 long fileSize) {
624 this.clientId = clientId;
625 this.offsets = offsets;
626 this.hbr = hbr;
627 this.types = types;
628 this.fileSize = fileSize;
629 }
630
631 @Override
632 public Boolean call() throws Exception {
633 Random rand = new Random(clientId.hashCode());
634 long endTime = System.currentTimeMillis() + 10000;
635 int numBlocksRead = 0;
636 int numPositionalRead = 0;
637 int numWithOnDiskSize = 0;
638 while (System.currentTimeMillis() < endTime) {
639 int blockId = rand.nextInt(NUM_TEST_BLOCKS);
640 long offset = offsets.get(blockId);
641 boolean pread = rand.nextBoolean();
642 boolean withOnDiskSize = rand.nextBoolean();
643 long expectedSize =
644 (blockId == NUM_TEST_BLOCKS - 1 ? fileSize
645 : offsets.get(blockId + 1)) - offset;
646
647 HFileBlock b;
648 try {
649 long onDiskSizeArg = withOnDiskSize ? expectedSize : -1;
650 b = hbr.readBlockData(offset, onDiskSizeArg, -1, pread);
651 } catch (IOException ex) {
652 LOG.error("Error in client " + clientId + " trying to read block at "
653 + offset + ", pread=" + pread + ", withOnDiskSize=" +
654 withOnDiskSize, ex);
655 return false;
656 }
657
658 assertEquals(types.get(blockId), b.getBlockType());
659 assertEquals(expectedSize, b.getOnDiskSizeWithHeader());
660 assertEquals(offset, b.getOffset());
661
662 ++numBlocksRead;
663 if (pread)
664 ++numPositionalRead;
665 if (withOnDiskSize)
666 ++numWithOnDiskSize;
667 }
668 LOG.info("Client " + clientId + " successfully read " + numBlocksRead +
669 " blocks (with pread: " + numPositionalRead + ", with onDiskSize " +
670 "specified: " + numWithOnDiskSize + ")");
671
672 return true;
673 }
674
675 }
676
677 @Test
678 public void testConcurrentReading() throws Exception {
679 for (Compression.Algorithm compressAlgo : COMPRESSION_ALGORITHMS) {
680 Path path =
681 new Path(TEST_UTIL.getDataTestDir(), "concurrent_reading");
682 Random rand = defaultRandom();
683 List<Long> offsets = new ArrayList<Long>();
684 List<BlockType> types = new ArrayList<BlockType>();
685 writeBlocks(rand, compressAlgo, path, offsets, null, types, null);
686 FSDataInputStream is = fs.open(path);
687 long fileSize = fs.getFileStatus(path).getLen();
688 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, compressAlgo,
689 fileSize);
690
691 Executor exec = Executors.newFixedThreadPool(NUM_READER_THREADS);
692 ExecutorCompletionService<Boolean> ecs =
693 new ExecutorCompletionService<Boolean>(exec);
694
695 for (int i = 0; i < NUM_READER_THREADS; ++i) {
696 ecs.submit(new BlockReaderThread("reader_" + (char) ('A' + i), hbr,
697 offsets, types, fileSize));
698 }
699
700 for (int i = 0; i < NUM_READER_THREADS; ++i) {
701 Future<Boolean> result = ecs.take();
702 assertTrue(result.get());
703 if (detailedLogging) {
704 LOG.info(String.valueOf(i + 1)
705 + " reader threads finished successfully (algo=" + compressAlgo
706 + ")");
707 }
708 }
709
710 is.close();
711 }
712 }
713
714 private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
715 Path path, List<Long> expectedOffsets, List<Long> expectedPrevOffsets,
716 List<BlockType> expectedTypes, List<ByteBuffer> expectedContents
717 ) throws IOException {
718 boolean cacheOnWrite = expectedContents != null;
719 FSDataOutputStream os = fs.create(path);
720 HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null,
721 includesMemstoreTS,
722 HFileReaderV2.MAX_MINOR_VERSION,
723 HFile.DEFAULT_CHECKSUM_TYPE,
724 HFile.DEFAULT_BYTES_PER_CHECKSUM);
725 Map<BlockType, Long> prevOffsetByType = new HashMap<BlockType, Long>();
726 long totalSize = 0;
727 for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
728 long pos = os.getPos();
729 int blockTypeOrdinal = rand.nextInt(BlockType.values().length);
730 if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) {
731 blockTypeOrdinal = BlockType.DATA.ordinal();
732 }
733 BlockType bt = BlockType.values()[blockTypeOrdinal];
734 DataOutputStream dos = hbw.startWriting(bt);
735 int size = rand.nextInt(500);
736 for (int j = 0; j < size; ++j) {
737
738 dos.writeShort(i + 1);
739 dos.writeInt(j + 1);
740 }
741
742 if (expectedOffsets != null)
743 expectedOffsets.add(os.getPos());
744
745 if (expectedPrevOffsets != null) {
746 Long prevOffset = prevOffsetByType.get(bt);
747 expectedPrevOffsets.add(prevOffset != null ? prevOffset : -1);
748 prevOffsetByType.put(bt, os.getPos());
749 }
750
751 expectedTypes.add(bt);
752
753 hbw.writeHeaderAndData(os);
754 totalSize += hbw.getOnDiskSizeWithHeader();
755
756 if (cacheOnWrite)
757 expectedContents.add(hbw.getUncompressedBufferWithHeader());
758
759 if (detailedLogging) {
760 LOG.info("Written block #" + i + " of type " + bt
761 + ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader()
762 + " at offset " + pos);
763 }
764 }
765 os.close();
766 LOG.info("Created a temporary file at " + path + ", "
767 + fs.getFileStatus(path).getLen() + " byte, compression=" +
768 compressAlgo);
769 return totalSize;
770 }
771
772 @Test
773 public void testBlockHeapSize() {
774 if (ClassSize.is32BitJVM()) {
775 assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 64);
776 } else {
777 assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 80);
778 }
779
780 for (int size : new int[] { 100, 256, 12345 }) {
781 byte[] byteArr = new byte[HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + size];
782 ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
783 HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
784 HFileBlock.FILL_HEADER, -1, includesMemstoreTS,
785 HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(),
786 0);
787 long byteBufferExpectedSize =
788 ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
789 + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + size);
790 long hfileBlockExpectedSize =
791 ClassSize.align(ClassSize.estimateBase(HFileBlock.class, true));
792 long expected = hfileBlockExpectedSize + byteBufferExpectedSize;
793 assertEquals("Block data size: " + size + ", byte buffer expected " +
794 "size: " + byteBufferExpectedSize + ", HFileBlock class expected " +
795 "size: " + hfileBlockExpectedSize + ";", expected,
796 block.heapSize());
797 }
798 }
799
800
801 @org.junit.Rule
802 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
803 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
804 }
805