1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import static org.junit.Assert.*;
22
23 import java.io.ByteArrayOutputStream;
24 import java.io.DataOutputStream;
25 import java.io.IOException;
26 import java.io.OutputStream;
27 import java.nio.ByteBuffer;
28 import java.util.ArrayList;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.HashMap;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Random;
35 import java.util.concurrent.Callable;
36 import java.util.concurrent.Executor;
37 import java.util.concurrent.ExecutorCompletionService;
38 import java.util.concurrent.Executors;
39 import java.util.concurrent.Future;
40
41 import org.apache.commons.logging.Log;
42 import org.apache.commons.logging.LogFactory;
43 import org.apache.hadoop.fs.FSDataInputStream;
44 import org.apache.hadoop.fs.FSDataOutputStream;
45 import org.apache.hadoop.fs.FileSystem;
46 import org.apache.hadoop.fs.Path;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HConstants;
49 import org.apache.hadoop.hbase.MediumTests;
50 import org.apache.hadoop.hbase.KeyValue;
51 import org.apache.hadoop.hbase.fs.HFileSystem;
52 import org.apache.hadoop.hbase.io.DoubleOutputStream;
53 import org.apache.hadoop.hbase.io.compress.Compression;
54 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
55 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
56 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
57 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
58 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
59 import org.apache.hadoop.hbase.util.Bytes;
60 import org.apache.hadoop.hbase.util.ChecksumType;
61 import org.apache.hadoop.hbase.util.ClassSize;
62 import org.apache.hadoop.io.WritableUtils;
63 import org.apache.hadoop.io.compress.Compressor;
64
65 import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.*;
66
67 import org.junit.Before;
68 import org.junit.Test;
69 import org.junit.experimental.categories.Category;
70 import org.junit.runner.RunWith;
71 import org.junit.runners.Parameterized;
72 import org.junit.runners.Parameterized.Parameters;
73
74 @Category(MediumTests.class)
75 @RunWith(Parameterized.class)
76 public class TestHFileBlock {
77
78 private static final boolean detailedLogging = false;
79 private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
80
81 private static final Log LOG = LogFactory.getLog(TestHFileBlock.class);
82
83 static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
84 NONE, GZ };
85
86 private static final int NUM_TEST_BLOCKS = 1000;
87 private static final int NUM_READER_THREADS = 26;
88
89
90 private static int NUM_KEYVALUES = 50;
91 private static int FIELD_LENGTH = 10;
92 private static float CHANCE_TO_REPEAT = 0.6f;
93
94 private static final HBaseTestingUtility TEST_UTIL =
95 new HBaseTestingUtility();
96 private FileSystem fs;
97 private int uncompressedSizeV1;
98
99 private final boolean includesMemstoreTS;
100
101 public TestHFileBlock(boolean includesMemstoreTS) {
102 this.includesMemstoreTS = includesMemstoreTS;
103 }
104
105 @Parameters
106 public static Collection<Object[]> parameters() {
107 return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
108 }
109
110 @Before
111 public void setUp() throws IOException {
112 fs = HFileSystem.get(TEST_UTIL.getConfiguration());
113 }
114
115 static void writeTestBlockContents(DataOutputStream dos) throws IOException {
116
117 for (int i = 0; i < 1000; ++i)
118 dos.writeInt(i / 100);
119 }
120
121 static int writeTestKeyValues(OutputStream dos, int seed, boolean includesMemstoreTS)
122 throws IOException {
123 List<KeyValue> keyValues = new ArrayList<KeyValue>();
124 Random randomizer = new Random(42l + seed);
125
126
127 for (int i = 0; i < NUM_KEYVALUES; ++i) {
128 byte[] row;
129 long timestamp;
130 byte[] family;
131 byte[] qualifier;
132 byte[] value;
133
134
135 if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
136 row = keyValues.get(randomizer.nextInt(keyValues.size())).getRow();
137 } else {
138 row = new byte[FIELD_LENGTH];
139 randomizer.nextBytes(row);
140 }
141 if (0 == i) {
142 family = new byte[FIELD_LENGTH];
143 randomizer.nextBytes(family);
144 } else {
145 family = keyValues.get(0).getFamily();
146 }
147 if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
148 qualifier = keyValues.get(
149 randomizer.nextInt(keyValues.size())).getQualifier();
150 } else {
151 qualifier = new byte[FIELD_LENGTH];
152 randomizer.nextBytes(qualifier);
153 }
154 if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
155 value = keyValues.get(randomizer.nextInt(keyValues.size())).getValue();
156 } else {
157 value = new byte[FIELD_LENGTH];
158 randomizer.nextBytes(value);
159 }
160 if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
161 timestamp = keyValues.get(
162 randomizer.nextInt(keyValues.size())).getTimestamp();
163 } else {
164 timestamp = randomizer.nextLong();
165 }
166
167 keyValues.add(new KeyValue(row, family, qualifier, timestamp, value));
168 }
169
170
171 int totalSize = 0;
172 Collections.sort(keyValues, KeyValue.COMPARATOR);
173 DataOutputStream dataOutputStream = new DataOutputStream(dos);
174 for (KeyValue kv : keyValues) {
175 totalSize += kv.getLength();
176 dataOutputStream.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
177 if (includesMemstoreTS) {
178 long memstoreTS = randomizer.nextLong();
179 WritableUtils.writeVLong(dataOutputStream, memstoreTS);
180 totalSize += WritableUtils.getVIntSize(memstoreTS);
181 }
182 }
183
184 return totalSize;
185 }
186
187 public byte[] createTestV1Block(Compression.Algorithm algo)
188 throws IOException {
189 Compressor compressor = algo.getCompressor();
190 ByteArrayOutputStream baos = new ByteArrayOutputStream();
191 OutputStream os = algo.createCompressionStream(baos, compressor, 0);
192 DataOutputStream dos = new DataOutputStream(os);
193 BlockType.META.write(dos);
194 writeTestBlockContents(dos);
195 uncompressedSizeV1 = dos.size();
196 dos.flush();
197 algo.returnCompressor(compressor);
198 return baos.toByteArray();
199 }
200
201 static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
202 boolean includesMemstoreTS) throws IOException {
203 final BlockType blockType = BlockType.DATA;
204 HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
205 includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
206 HFile.DEFAULT_BYTES_PER_CHECKSUM);
207 DataOutputStream dos = hbw.startWriting(blockType);
208 writeTestBlockContents(dos);
209 dos.flush();
210 byte[] headerAndData = hbw.getHeaderAndDataForTest();
211 assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
212 hbw.release();
213 return hbw;
214 }
215
216 public String createTestBlockStr(Compression.Algorithm algo,
217 int correctLength) throws IOException {
218 HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS);
219 byte[] testV2Block = hbw.getHeaderAndDataForTest();
220 int osOffset = HConstants.HFILEBLOCK_HEADER_SIZE + 9;
221 if (testV2Block.length == correctLength) {
222
223
224
225
226
227 testV2Block[osOffset] = 3;
228 }
229 return Bytes.toStringBinary(testV2Block);
230 }
231
232 @Test
233 public void testNoCompression() throws IOException {
234 assertEquals(4000, createTestV2Block(NONE, includesMemstoreTS).
235 getBlockForCaching().getUncompressedSizeWithoutHeader());
236 }
237
238 @Test
239 public void testGzipCompression() throws IOException {
240 final String correctTestBlockStr =
241 "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
242 + "\\xFF\\xFF\\xFF\\xFF"
243 + "\\x01\\x00\\x00@\\x00\\x00\\x00\\x00["
244
245 + "\\x1F\\x8B"
246 + "\\x08"
247 + "\\x00"
248 + "\\x00\\x00\\x00\\x00"
249 + "\\x00"
250
251
252
253
254 + "\\x03"
255 + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
256 + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
257 + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00"
258 + "\\x00\\x00\\x00\\x00";
259 final int correctGzipBlockLength = 95;
260 final String testBlockStr = createTestBlockStr(GZ, correctGzipBlockLength);
261
262
263 assertEquals(correctTestBlockStr.substring(0, correctGzipBlockLength - 4),
264 testBlockStr.substring(0, correctGzipBlockLength - 4));
265 }
266
267 @Test
268 public void testReaderV2() throws IOException {
269 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
270 for (boolean pread : new boolean[] { false, true }) {
271 LOG.info("testReaderV2: Compression algorithm: " + algo +
272 ", pread=" + pread);
273 Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
274 + algo);
275 FSDataOutputStream os = fs.create(path);
276 HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
277 includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
278 HFile.DEFAULT_BYTES_PER_CHECKSUM);
279 long totalSize = 0;
280 for (int blockId = 0; blockId < 2; ++blockId) {
281 DataOutputStream dos = hbw.startWriting(BlockType.DATA);
282 for (int i = 0; i < 1234; ++i)
283 dos.writeInt(i);
284 hbw.writeHeaderAndData(os);
285 totalSize += hbw.getOnDiskSizeWithHeader();
286 }
287 os.close();
288
289 FSDataInputStream is = fs.open(path);
290 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo,
291 totalSize);
292 HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
293 is.close();
294 assertEquals(0, HFile.getChecksumFailuresCount());
295
296 b.sanityCheck();
297 assertEquals(4936, b.getUncompressedSizeWithoutHeader());
298 assertEquals(algo == GZ ? 2173 : 4936,
299 b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
300 String blockStr = b.toString();
301
302 if (algo == GZ) {
303 is = fs.open(path);
304 hbr = new HFileBlock.FSReaderV2(is, algo, totalSize);
305 b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE +
306 b.totalChecksumBytes(), -1, pread);
307 assertEquals(blockStr, b.toString());
308 int wrongCompressedSize = 2172;
309 try {
310 b = hbr.readBlockData(0, wrongCompressedSize
311 + HConstants.HFILEBLOCK_HEADER_SIZE, -1, pread);
312 fail("Exception expected");
313 } catch (IOException ex) {
314 String expectedPrefix = "On-disk size without header provided is "
315 + wrongCompressedSize + ", but block header contains "
316 + b.getOnDiskSizeWithoutHeader() + ".";
317 assertTrue("Invalid exception message: '" + ex.getMessage()
318 + "'.\nMessage is expected to start with: '" + expectedPrefix
319 + "'", ex.getMessage().startsWith(expectedPrefix));
320 }
321 is.close();
322 }
323 }
324 }
325 }
326
327
328
329
330
331 @Test
332 public void testDataBlockEncoding() throws IOException {
333 final int numBlocks = 5;
334 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
335 for (boolean pread : new boolean[] { false, true }) {
336 for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
337 Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
338 + algo + "_" + encoding.toString());
339 FSDataOutputStream os = fs.create(path);
340 HFileDataBlockEncoder dataBlockEncoder =
341 new HFileDataBlockEncoderImpl(encoding);
342 HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder,
343 includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
344 HFile.DEFAULT_BYTES_PER_CHECKSUM);
345 long totalSize = 0;
346 final List<Integer> encodedSizes = new ArrayList<Integer>();
347 final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
348 for (int blockId = 0; blockId < numBlocks; ++blockId) {
349 DataOutputStream dos = hbw.startWriting(BlockType.DATA);
350 writeEncodedBlock(algo, encoding, dos, encodedSizes, encodedBlocks,
351 blockId, includesMemstoreTS, HConstants.HFILEBLOCK_DUMMY_HEADER);
352 hbw.writeHeaderAndData(os);
353 totalSize += hbw.getOnDiskSizeWithHeader();
354 }
355 os.close();
356
357 FSDataInputStream is = fs.open(path);
358 HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, algo,
359 totalSize);
360 hbr.setDataBlockEncoder(dataBlockEncoder);
361 hbr.setIncludesMemstoreTS(includesMemstoreTS);
362
363 HFileBlock b;
364 int pos = 0;
365 for (int blockId = 0; blockId < numBlocks; ++blockId) {
366 b = hbr.readBlockData(pos, -1, -1, pread);
367 assertEquals(0, HFile.getChecksumFailuresCount());
368 b.sanityCheck();
369 pos += b.getOnDiskSizeWithHeader();
370 assertEquals((int) encodedSizes.get(blockId),
371 b.getUncompressedSizeWithoutHeader());
372 ByteBuffer actualBuffer = b.getBufferWithoutHeader();
373 if (encoding != DataBlockEncoding.NONE) {
374
375 assertEquals(0, actualBuffer.get(0));
376 assertEquals(encoding.getId(), actualBuffer.get(1));
377 actualBuffer.position(2);
378 actualBuffer = actualBuffer.slice();
379 }
380
381 ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
382 expectedBuffer.rewind();
383
384
385 assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding,
386 pread);
387 }
388 is.close();
389 }
390 }
391 }
392 }
393
394 static void writeEncodedBlock(Algorithm algo, DataBlockEncoding encoding,
395 DataOutputStream dos, final List<Integer> encodedSizes,
396 final List<ByteBuffer> encodedBlocks, int blockId,
397 boolean includesMemstoreTS, byte[] dummyHeader) throws IOException {
398 ByteArrayOutputStream baos = new ByteArrayOutputStream();
399 DoubleOutputStream doubleOutputStream =
400 new DoubleOutputStream(dos, baos);
401 writeTestKeyValues(doubleOutputStream, blockId, includesMemstoreTS);
402 ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
403 rawBuf.rewind();
404
405 DataBlockEncoder encoder = encoding.getEncoder();
406 int headerLen = dummyHeader.length;
407 byte[] encodedResultWithHeader = null;
408 if (encoder != null) {
409 HFileBlockEncodingContext encodingCtx =
410 encoder.newDataBlockEncodingContext(algo, encoding, dummyHeader);
411 encoder.encodeKeyValues(rawBuf, includesMemstoreTS,
412 encodingCtx);
413 encodedResultWithHeader =
414 encodingCtx.getUncompressedBytesWithHeader();
415 } else {
416 HFileBlockDefaultEncodingContext defaultEncodingCtx =
417 new HFileBlockDefaultEncodingContext(algo, encoding, dummyHeader);
418 byte[] rawBufWithHeader =
419 new byte[rawBuf.array().length + headerLen];
420 System.arraycopy(rawBuf.array(), 0, rawBufWithHeader,
421 headerLen, rawBuf.array().length);
422 defaultEncodingCtx.compressAfterEncodingWithBlockType(rawBufWithHeader,
423 BlockType.DATA);
424 encodedResultWithHeader =
425 defaultEncodingCtx.getUncompressedBytesWithHeader();
426 }
427 final int encodedSize =
428 encodedResultWithHeader.length - headerLen;
429 if (encoder != null) {
430
431
432 headerLen += DataBlockEncoding.ID_SIZE;
433 }
434 byte[] encodedDataSection =
435 new byte[encodedResultWithHeader.length - headerLen];
436 System.arraycopy(encodedResultWithHeader, headerLen,
437 encodedDataSection, 0, encodedDataSection.length);
438 final ByteBuffer encodedBuf =
439 ByteBuffer.wrap(encodedDataSection);
440 encodedSizes.add(encodedSize);
441 encodedBlocks.add(encodedBuf);
442 }
443
444 static void assertBuffersEqual(ByteBuffer expectedBuffer,
445 ByteBuffer actualBuffer, Compression.Algorithm compression,
446 DataBlockEncoding encoding, boolean pread) {
447 if (!actualBuffer.equals(expectedBuffer)) {
448 int prefix = 0;
449 int minLimit = Math.min(expectedBuffer.limit(), actualBuffer.limit());
450 while (prefix < minLimit &&
451 expectedBuffer.get(prefix) == actualBuffer.get(prefix)) {
452 prefix++;
453 }
454
455 fail(String.format(
456 "Content mismath for compression %s, encoding %s, " +
457 "pread %s, commonPrefix %d, expected %s, got %s",
458 compression, encoding, pread, prefix,
459 nextBytesToStr(expectedBuffer, prefix),
460 nextBytesToStr(actualBuffer, prefix)));
461 }
462 }
463
464
465
466
467
468 private static String nextBytesToStr(ByteBuffer buf, int pos) {
469 int maxBytes = buf.limit() - pos;
470 int numBytes = Math.min(16, maxBytes);
471 return Bytes.toStringBinary(buf.array(), buf.arrayOffset() + pos,
472 numBytes) + (numBytes < maxBytes ? "..." : "");
473 }
474
475 @Test
476 public void testPreviousOffset() throws IOException {
477 for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
478 for (boolean pread : BOOLEAN_VALUES) {
479 for (boolean cacheOnWrite : BOOLEAN_VALUES) {
480 Random rand = defaultRandom();
481 LOG.info("testPreviousOffset:Compression algorithm: " + algo +
482 ", pread=" + pread +
483 ", cacheOnWrite=" + cacheOnWrite);
484 Path path = new Path(TEST_UTIL.getDataTestDir(), "prev_offset");
485 List<Long> expectedOffsets = new ArrayList<Long>();
486 List<Long> expectedPrevOffsets = new ArrayList<Long>();
487 List<BlockType> expectedTypes = new ArrayList<BlockType>();
488 List<ByteBuffer> expectedContents = cacheOnWrite
489 ? new ArrayList<ByteBuffer>() : null;
490 long totalSize = writeBlocks(rand, algo, path, expectedOffsets,
491 expectedPrevOffsets, expectedTypes, expectedContents);
492
493 FSDataInputStream is = fs.open(path);
494 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo,
495 totalSize);
496 long curOffset = 0;
497 for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
498 if (!pread) {
499 assertEquals(is.getPos(), curOffset + (i == 0 ? 0 :
500 HConstants.HFILEBLOCK_HEADER_SIZE));
501 }
502
503 assertEquals(expectedOffsets.get(i).longValue(), curOffset);
504 if (detailedLogging) {
505 LOG.info("Reading block #" + i + " at offset " + curOffset);
506 }
507 HFileBlock b = hbr.readBlockData(curOffset, -1, -1, pread);
508 if (detailedLogging) {
509 LOG.info("Block #" + i + ": " + b);
510 }
511 assertEquals("Invalid block #" + i + "'s type:",
512 expectedTypes.get(i), b.getBlockType());
513 assertEquals("Invalid previous block offset for block " + i
514 + " of " + "type " + b.getBlockType() + ":",
515 (long) expectedPrevOffsets.get(i), b.getPrevBlockOffset());
516 b.sanityCheck();
517 assertEquals(curOffset, b.getOffset());
518
519
520
521 HFileBlock b2 = hbr.readBlockData(curOffset,
522 b.getOnDiskSizeWithHeader(), -1, pread);
523 b2.sanityCheck();
524
525 assertEquals(b.getBlockType(), b2.getBlockType());
526 assertEquals(b.getOnDiskSizeWithoutHeader(),
527 b2.getOnDiskSizeWithoutHeader());
528 assertEquals(b.getOnDiskSizeWithHeader(),
529 b2.getOnDiskSizeWithHeader());
530 assertEquals(b.getUncompressedSizeWithoutHeader(),
531 b2.getUncompressedSizeWithoutHeader());
532 assertEquals(b.getPrevBlockOffset(), b2.getPrevBlockOffset());
533 assertEquals(curOffset, b2.getOffset());
534 assertEquals(b.getBytesPerChecksum(), b2.getBytesPerChecksum());
535 assertEquals(b.getOnDiskDataSizeWithHeader(),
536 b2.getOnDiskDataSizeWithHeader());
537 assertEquals(0, HFile.getChecksumFailuresCount());
538
539 curOffset += b.getOnDiskSizeWithHeader();
540
541 if (cacheOnWrite) {
542
543
544
545
546 ByteBuffer bufRead = b.getBufferWithHeader();
547 ByteBuffer bufExpected = expectedContents.get(i);
548 boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
549 bufRead.arrayOffset(),
550 bufRead.limit() - b.totalChecksumBytes(),
551 bufExpected.array(), bufExpected.arrayOffset(),
552 bufExpected.limit()) == 0;
553 String wrongBytesMsg = "";
554
555 if (!bytesAreCorrect) {
556
557
558 wrongBytesMsg = "Expected bytes in block #" + i + " (algo="
559 + algo + ", pread=" + pread
560 + ", cacheOnWrite=" + cacheOnWrite + "):\n";
561 wrongBytesMsg += Bytes.toStringBinary(bufExpected.array(),
562 bufExpected.arrayOffset(), Math.min(32,
563 bufExpected.limit()))
564 + ", actual:\n"
565 + Bytes.toStringBinary(bufRead.array(),
566 bufRead.arrayOffset(), Math.min(32, bufRead.limit()));
567 if (detailedLogging) {
568 LOG.warn("expected header" +
569 HFileBlock.toStringHeader(bufExpected) +
570 "\nfound header" +
571 HFileBlock.toStringHeader(bufRead));
572 LOG.warn("bufread offset " + bufRead.arrayOffset() +
573 " limit " + bufRead.limit() +
574 " expected offset " + bufExpected.arrayOffset() +
575 " limit " + bufExpected.limit());
576 LOG.warn(wrongBytesMsg);
577 }
578 }
579 assertTrue(wrongBytesMsg, bytesAreCorrect);
580 }
581 }
582
583 assertEquals(curOffset, fs.getFileStatus(path).getLen());
584 is.close();
585 }
586 }
587 }
588 }
589
590 private Random defaultRandom() {
591 return new Random(189237);
592 }
593
594 private class BlockReaderThread implements Callable<Boolean> {
595 private final String clientId;
596 private final HFileBlock.FSReader hbr;
597 private final List<Long> offsets;
598 private final List<BlockType> types;
599 private final long fileSize;
600
601 public BlockReaderThread(String clientId,
602 HFileBlock.FSReader hbr, List<Long> offsets, List<BlockType> types,
603 long fileSize) {
604 this.clientId = clientId;
605 this.offsets = offsets;
606 this.hbr = hbr;
607 this.types = types;
608 this.fileSize = fileSize;
609 }
610
611 @Override
612 public Boolean call() throws Exception {
613 Random rand = new Random(clientId.hashCode());
614 long endTime = System.currentTimeMillis() + 10000;
615 int numBlocksRead = 0;
616 int numPositionalRead = 0;
617 int numWithOnDiskSize = 0;
618 while (System.currentTimeMillis() < endTime) {
619 int blockId = rand.nextInt(NUM_TEST_BLOCKS);
620 long offset = offsets.get(blockId);
621 boolean pread = rand.nextBoolean();
622 boolean withOnDiskSize = rand.nextBoolean();
623 long expectedSize =
624 (blockId == NUM_TEST_BLOCKS - 1 ? fileSize
625 : offsets.get(blockId + 1)) - offset;
626
627 HFileBlock b;
628 try {
629 long onDiskSizeArg = withOnDiskSize ? expectedSize : -1;
630 b = hbr.readBlockData(offset, onDiskSizeArg, -1, pread);
631 } catch (IOException ex) {
632 LOG.error("Error in client " + clientId + " trying to read block at "
633 + offset + ", pread=" + pread + ", withOnDiskSize=" +
634 withOnDiskSize, ex);
635 return false;
636 }
637
638 assertEquals(types.get(blockId), b.getBlockType());
639 assertEquals(expectedSize, b.getOnDiskSizeWithHeader());
640 assertEquals(offset, b.getOffset());
641
642 ++numBlocksRead;
643 if (pread)
644 ++numPositionalRead;
645 if (withOnDiskSize)
646 ++numWithOnDiskSize;
647 }
648 LOG.info("Client " + clientId + " successfully read " + numBlocksRead +
649 " blocks (with pread: " + numPositionalRead + ", with onDiskSize " +
650 "specified: " + numWithOnDiskSize + ")");
651
652 return true;
653 }
654
655 }
656
657 @Test
658 public void testConcurrentReading() throws Exception {
659 for (Compression.Algorithm compressAlgo : COMPRESSION_ALGORITHMS) {
660 Path path =
661 new Path(TEST_UTIL.getDataTestDir(), "concurrent_reading");
662 Random rand = defaultRandom();
663 List<Long> offsets = new ArrayList<Long>();
664 List<BlockType> types = new ArrayList<BlockType>();
665 writeBlocks(rand, compressAlgo, path, offsets, null, types, null);
666 FSDataInputStream is = fs.open(path);
667 long fileSize = fs.getFileStatus(path).getLen();
668 HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, compressAlgo,
669 fileSize);
670
671 Executor exec = Executors.newFixedThreadPool(NUM_READER_THREADS);
672 ExecutorCompletionService<Boolean> ecs =
673 new ExecutorCompletionService<Boolean>(exec);
674
675 for (int i = 0; i < NUM_READER_THREADS; ++i) {
676 ecs.submit(new BlockReaderThread("reader_" + (char) ('A' + i), hbr,
677 offsets, types, fileSize));
678 }
679
680 for (int i = 0; i < NUM_READER_THREADS; ++i) {
681 Future<Boolean> result = ecs.take();
682 assertTrue(result.get());
683 if (detailedLogging) {
684 LOG.info(String.valueOf(i + 1)
685 + " reader threads finished successfully (algo=" + compressAlgo
686 + ")");
687 }
688 }
689
690 is.close();
691 }
692 }
693
694 private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
695 Path path, List<Long> expectedOffsets, List<Long> expectedPrevOffsets,
696 List<BlockType> expectedTypes, List<ByteBuffer> expectedContents
697 ) throws IOException {
698 boolean cacheOnWrite = expectedContents != null;
699 FSDataOutputStream os = fs.create(path);
700 HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null,
701 includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
702 HFile.DEFAULT_BYTES_PER_CHECKSUM);
703 Map<BlockType, Long> prevOffsetByType = new HashMap<BlockType, Long>();
704 long totalSize = 0;
705 for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
706 long pos = os.getPos();
707 int blockTypeOrdinal = rand.nextInt(BlockType.values().length);
708 if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) {
709 blockTypeOrdinal = BlockType.DATA.ordinal();
710 }
711 BlockType bt = BlockType.values()[blockTypeOrdinal];
712 DataOutputStream dos = hbw.startWriting(bt);
713 int size = rand.nextInt(500);
714 for (int j = 0; j < size; ++j) {
715
716 dos.writeShort(i + 1);
717 dos.writeInt(j + 1);
718 }
719
720 if (expectedOffsets != null)
721 expectedOffsets.add(os.getPos());
722
723 if (expectedPrevOffsets != null) {
724 Long prevOffset = prevOffsetByType.get(bt);
725 expectedPrevOffsets.add(prevOffset != null ? prevOffset : -1);
726 prevOffsetByType.put(bt, os.getPos());
727 }
728
729 expectedTypes.add(bt);
730
731 hbw.writeHeaderAndData(os);
732 totalSize += hbw.getOnDiskSizeWithHeader();
733
734 if (cacheOnWrite)
735 expectedContents.add(hbw.getUncompressedBufferWithHeader());
736
737 if (detailedLogging) {
738 LOG.info("Written block #" + i + " of type " + bt
739 + ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader()
740 + " at offset " + pos);
741 }
742 }
743 os.close();
744 LOG.info("Created a temporary file at " + path + ", "
745 + fs.getFileStatus(path).getLen() + " byte, compression=" +
746 compressAlgo);
747 return totalSize;
748 }
749
750 @Test
751 public void testBlockHeapSize() {
752 if (ClassSize.is32BitJVM()) {
753 assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 64);
754 } else {
755 assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 80);
756 }
757
758 for (int size : new int[] { 100, 256, 12345 }) {
759 byte[] byteArr = new byte[HConstants.HFILEBLOCK_HEADER_SIZE + size];
760 ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
761 HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
762 HFileBlock.FILL_HEADER, -1, includesMemstoreTS,
763 HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(),
764 0);
765 long byteBufferExpectedSize =
766 ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
767 + HConstants.HFILEBLOCK_HEADER_SIZE + size);
768 long hfileBlockExpectedSize =
769 ClassSize.align(ClassSize.estimateBase(HFileBlock.class, true));
770 long expected = hfileBlockExpectedSize + byteBufferExpectedSize;
771 assertEquals("Block data size: " + size + ", byte buffer expected " +
772 "size: " + byteBufferExpectedSize + ", HFileBlock class expected " +
773 "size: " + hfileBlockExpectedSize + ";", expected,
774 block.heapSize());
775 }
776 }
777
778
779 }
780