View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import static org.junit.Assert.*;
22  
23  import java.io.ByteArrayOutputStream;
24  import java.io.DataOutputStream;
25  import java.io.IOException;
26  import java.io.OutputStream;
27  import java.nio.ByteBuffer;
28  import java.util.ArrayList;
29  import java.util.Collection;
30  import java.util.Collections;
31  import java.util.HashMap;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.Random;
35  import java.util.concurrent.Callable;
36  import java.util.concurrent.Executor;
37  import java.util.concurrent.ExecutorCompletionService;
38  import java.util.concurrent.Executors;
39  import java.util.concurrent.Future;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.fs.FSDataInputStream;
44  import org.apache.hadoop.fs.FSDataOutputStream;
45  import org.apache.hadoop.fs.FileSystem;
46  import org.apache.hadoop.fs.Path;
47  import org.apache.hadoop.hbase.HBaseTestingUtility;
48  import org.apache.hadoop.hbase.HConstants;
49  import org.apache.hadoop.hbase.MediumTests;
50  import org.apache.hadoop.hbase.KeyValue;
51  import org.apache.hadoop.hbase.fs.HFileSystem;
52  import org.apache.hadoop.hbase.io.DoubleOutputStream;
53  import org.apache.hadoop.hbase.io.compress.Compression;
54  import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
55  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
56  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
57  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
58  import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
59  import org.apache.hadoop.hbase.util.Bytes;
60  import org.apache.hadoop.hbase.util.ChecksumType;
61  import org.apache.hadoop.hbase.util.ClassSize;
62  import org.apache.hadoop.io.WritableUtils;
63  import org.apache.hadoop.io.compress.Compressor;
64  
65  import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.*;
66  
67  import org.junit.Before;
68  import org.junit.Test;
69  import org.junit.experimental.categories.Category;
70  import org.junit.runner.RunWith;
71  import org.junit.runners.Parameterized;
72  import org.junit.runners.Parameterized.Parameters;
73  
74  @Category(MediumTests.class)
75  @RunWith(Parameterized.class)
76  public class TestHFileBlock {
77    // change this value to activate more logs
78    private static final boolean detailedLogging = false;
79    private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
80  
81    private static final Log LOG = LogFactory.getLog(TestHFileBlock.class);
82  
83    static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
84        NONE, GZ };
85  
86    private static final int NUM_TEST_BLOCKS = 1000;
87    private static final int NUM_READER_THREADS = 26;
88  
89    // Used to generate KeyValues
90    private static int NUM_KEYVALUES = 50;
91    private static int FIELD_LENGTH = 10;
92    private static float CHANCE_TO_REPEAT = 0.6f;
93  
94    private static final HBaseTestingUtility TEST_UTIL =
95      new HBaseTestingUtility();
96    private FileSystem fs;
97    private int uncompressedSizeV1;
98  
99    private final boolean includesMemstoreTS;
100 
101   public TestHFileBlock(boolean includesMemstoreTS) {
102     this.includesMemstoreTS = includesMemstoreTS;
103   }
104 
105   @Parameters
106   public static Collection<Object[]> parameters() {
107     return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
108   }
109 
110   @Before
111   public void setUp() throws IOException {
112     fs = HFileSystem.get(TEST_UTIL.getConfiguration());
113   }
114 
115   static void writeTestBlockContents(DataOutputStream dos) throws IOException {
116     // This compresses really well.
117     for (int i = 0; i < 1000; ++i)
118       dos.writeInt(i / 100);
119   }
120 
121  static int writeTestKeyValues(OutputStream dos, int seed, boolean includesMemstoreTS)
122       throws IOException {
123     List<KeyValue> keyValues = new ArrayList<KeyValue>();
124     Random randomizer = new Random(42l + seed); // just any fixed number
125 
126     // generate keyValues
127     for (int i = 0; i < NUM_KEYVALUES; ++i) {
128       byte[] row;
129       long timestamp;
130       byte[] family;
131       byte[] qualifier;
132       byte[] value;
133 
134       // generate it or repeat, it should compress well
135       if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
136         row = keyValues.get(randomizer.nextInt(keyValues.size())).getRow();
137       } else {
138         row = new byte[FIELD_LENGTH];
139         randomizer.nextBytes(row);
140       }
141       if (0 == i) {
142         family = new byte[FIELD_LENGTH];
143         randomizer.nextBytes(family);
144       } else {
145         family = keyValues.get(0).getFamily();
146       }
147       if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
148         qualifier = keyValues.get(
149             randomizer.nextInt(keyValues.size())).getQualifier();
150       } else {
151         qualifier = new byte[FIELD_LENGTH];
152         randomizer.nextBytes(qualifier);
153       }
154       if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
155         value = keyValues.get(randomizer.nextInt(keyValues.size())).getValue();
156       } else {
157         value = new byte[FIELD_LENGTH];
158         randomizer.nextBytes(value);
159       }
160       if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
161         timestamp = keyValues.get(
162             randomizer.nextInt(keyValues.size())).getTimestamp();
163       } else {
164         timestamp = randomizer.nextLong();
165       }
166 
167       keyValues.add(new KeyValue(row, family, qualifier, timestamp, value));
168     }
169 
170     // sort it and write to stream
171     int totalSize = 0;
172     Collections.sort(keyValues, KeyValue.COMPARATOR);
173     DataOutputStream dataOutputStream = new DataOutputStream(dos);
174     for (KeyValue kv : keyValues) {
175       totalSize += kv.getLength();
176       dataOutputStream.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
177       if (includesMemstoreTS) {
178         long memstoreTS = randomizer.nextLong();
179         WritableUtils.writeVLong(dataOutputStream, memstoreTS);
180         totalSize += WritableUtils.getVIntSize(memstoreTS);
181       }
182     }
183 
184     return totalSize;
185   }
186 
187   public byte[] createTestV1Block(Compression.Algorithm algo)
188       throws IOException {
189     Compressor compressor = algo.getCompressor();
190     ByteArrayOutputStream baos = new ByteArrayOutputStream();
191     OutputStream os = algo.createCompressionStream(baos, compressor, 0);
192     DataOutputStream dos = new DataOutputStream(os);
193     BlockType.META.write(dos); // Let's make this a meta block.
194     writeTestBlockContents(dos);
195     uncompressedSizeV1 = dos.size();
196     dos.flush();
197     algo.returnCompressor(compressor);
198     return baos.toByteArray();
199   }
200 
201   static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
202       boolean includesMemstoreTS) throws IOException {
203     final BlockType blockType = BlockType.DATA;
204     HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
205         includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
206         HFile.DEFAULT_BYTES_PER_CHECKSUM);
207     DataOutputStream dos = hbw.startWriting(blockType);
208     writeTestBlockContents(dos);
209     dos.flush();
210     byte[] headerAndData = hbw.getHeaderAndDataForTest();
211     assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
212     hbw.release();
213     return hbw;
214   }
215 
216   public String createTestBlockStr(Compression.Algorithm algo,
217       int correctLength) throws IOException {
218     HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS);
219     byte[] testV2Block = hbw.getHeaderAndDataForTest();
220     int osOffset = HConstants.HFILEBLOCK_HEADER_SIZE + 9;
221     if (testV2Block.length == correctLength) {
222       // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
223       // variations across operating systems.
224       // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format.
225       // We only make this change when the compressed block length matches.
226       // Otherwise, there are obviously other inconsistencies.
227       testV2Block[osOffset] = 3;
228     }
229     return Bytes.toStringBinary(testV2Block);
230   }
231 
232   @Test
233   public void testNoCompression() throws IOException {
234     assertEquals(4000, createTestV2Block(NONE, includesMemstoreTS).
235                  getBlockForCaching().getUncompressedSizeWithoutHeader());
236   }
237 
238   @Test
239   public void testGzipCompression() throws IOException {
240     final String correctTestBlockStr =
241         "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
242             + "\\xFF\\xFF\\xFF\\xFF"
243             + "\\x01\\x00\\x00@\\x00\\x00\\x00\\x00["
244             // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
245             + "\\x1F\\x8B"  // gzip magic signature
246             + "\\x08"  // Compression method: 8 = "deflate"
247             + "\\x00"  // Flags
248             + "\\x00\\x00\\x00\\x00"  // mtime
249             + "\\x00"  // XFL (extra flags)
250             // OS (0 = FAT filesystems, 3 = Unix). However, this field
251             // sometimes gets set to 0 on Linux and Mac, so we reset it to 3.
252             // This appears to be a difference caused by the availability
253             // (and use) of the native GZ codec.
254             + "\\x03"
255             + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
256             + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
257             + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00"
258             + "\\x00\\x00\\x00\\x00"; //  4 byte checksum (ignored)
259     final int correctGzipBlockLength = 95;
260     final String testBlockStr = createTestBlockStr(GZ, correctGzipBlockLength);
261     // We ignore the block checksum because createTestBlockStr can change the
262     // gzip header after the block is produced
263     assertEquals(correctTestBlockStr.substring(0, correctGzipBlockLength - 4),
264       testBlockStr.substring(0, correctGzipBlockLength - 4));
265   }
266 
267   @Test
268   public void testReaderV2() throws IOException {
269     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
270       for (boolean pread : new boolean[] { false, true }) {
271           LOG.info("testReaderV2: Compression algorithm: " + algo +
272                    ", pread=" + pread);
273         Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
274             + algo);
275         FSDataOutputStream os = fs.create(path);
276         HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
277             includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
278             HFile.DEFAULT_BYTES_PER_CHECKSUM);
279         long totalSize = 0;
280         for (int blockId = 0; blockId < 2; ++blockId) {
281           DataOutputStream dos = hbw.startWriting(BlockType.DATA);
282           for (int i = 0; i < 1234; ++i)
283             dos.writeInt(i);
284           hbw.writeHeaderAndData(os);
285           totalSize += hbw.getOnDiskSizeWithHeader();
286         }
287         os.close();
288 
289         FSDataInputStream is = fs.open(path);
290         HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo,
291             totalSize);
292         HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
293         is.close();
294         assertEquals(0, HFile.getChecksumFailuresCount());
295 
296         b.sanityCheck();
297         assertEquals(4936, b.getUncompressedSizeWithoutHeader());
298         assertEquals(algo == GZ ? 2173 : 4936,
299                      b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
300         String blockStr = b.toString();
301 
302         if (algo == GZ) {
303           is = fs.open(path);
304           hbr = new HFileBlock.FSReaderV2(is, algo, totalSize);
305           b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE +
306                                 b.totalChecksumBytes(), -1, pread);
307           assertEquals(blockStr, b.toString());
308           int wrongCompressedSize = 2172;
309           try {
310             b = hbr.readBlockData(0, wrongCompressedSize
311                 + HConstants.HFILEBLOCK_HEADER_SIZE, -1, pread);
312             fail("Exception expected");
313           } catch (IOException ex) {
314             String expectedPrefix = "On-disk size without header provided is "
315                 + wrongCompressedSize + ", but block header contains "
316                 + b.getOnDiskSizeWithoutHeader() + ".";
317             assertTrue("Invalid exception message: '" + ex.getMessage()
318                 + "'.\nMessage is expected to start with: '" + expectedPrefix
319                 + "'", ex.getMessage().startsWith(expectedPrefix));
320           }
321           is.close();
322         }
323       }
324     }
325   }
326 
327   /**
328    * Test encoding/decoding data blocks.
329    * @throws IOException a bug or a problem with temporary files.
330    */
331   @Test
332   public void testDataBlockEncoding() throws IOException {
333     final int numBlocks = 5;
334     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
335       for (boolean pread : new boolean[] { false, true }) {
336         for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
337           Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
338               + algo + "_" + encoding.toString());
339           FSDataOutputStream os = fs.create(path);
340           HFileDataBlockEncoder dataBlockEncoder =
341               new HFileDataBlockEncoderImpl(encoding);
342           HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder,
343               includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
344               HFile.DEFAULT_BYTES_PER_CHECKSUM);
345           long totalSize = 0;
346           final List<Integer> encodedSizes = new ArrayList<Integer>();
347           final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
348           for (int blockId = 0; blockId < numBlocks; ++blockId) {
349             DataOutputStream dos = hbw.startWriting(BlockType.DATA);
350             writeEncodedBlock(algo, encoding, dos, encodedSizes, encodedBlocks,
351                 blockId, includesMemstoreTS, HConstants.HFILEBLOCK_DUMMY_HEADER);
352             hbw.writeHeaderAndData(os);
353             totalSize += hbw.getOnDiskSizeWithHeader();
354           }
355           os.close();
356 
357           FSDataInputStream is = fs.open(path);
358           HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, algo,
359               totalSize);
360           hbr.setDataBlockEncoder(dataBlockEncoder);
361           hbr.setIncludesMemstoreTS(includesMemstoreTS);
362 
363           HFileBlock b;
364           int pos = 0;
365           for (int blockId = 0; blockId < numBlocks; ++blockId) {
366             b = hbr.readBlockData(pos, -1, -1, pread);
367             assertEquals(0, HFile.getChecksumFailuresCount());
368             b.sanityCheck();
369             pos += b.getOnDiskSizeWithHeader();
370             assertEquals((int) encodedSizes.get(blockId),
371                 b.getUncompressedSizeWithoutHeader());
372             ByteBuffer actualBuffer = b.getBufferWithoutHeader();
373             if (encoding != DataBlockEncoding.NONE) {
374               // We expect a two-byte big-endian encoding id.
375               assertEquals(0, actualBuffer.get(0));
376               assertEquals(encoding.getId(), actualBuffer.get(1));
377               actualBuffer.position(2);
378               actualBuffer = actualBuffer.slice();
379             }
380 
381             ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
382             expectedBuffer.rewind();
383 
384             // test if content matches, produce nice message
385             assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding,
386                 pread);
387           }
388           is.close();
389         }
390       }
391     }
392   }
393 
394   static void writeEncodedBlock(Algorithm algo, DataBlockEncoding encoding,
395        DataOutputStream dos, final List<Integer> encodedSizes,
396       final List<ByteBuffer> encodedBlocks, int blockId,
397       boolean includesMemstoreTS, byte[] dummyHeader) throws IOException {
398     ByteArrayOutputStream baos = new ByteArrayOutputStream();
399     DoubleOutputStream doubleOutputStream =
400         new DoubleOutputStream(dos, baos);
401     writeTestKeyValues(doubleOutputStream, blockId, includesMemstoreTS);
402     ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
403     rawBuf.rewind();
404 
405     DataBlockEncoder encoder = encoding.getEncoder();
406     int headerLen = dummyHeader.length;
407     byte[] encodedResultWithHeader = null;
408     if (encoder != null) {
409       HFileBlockEncodingContext encodingCtx =
410           encoder.newDataBlockEncodingContext(algo, encoding, dummyHeader);
411       encoder.encodeKeyValues(rawBuf, includesMemstoreTS,
412           encodingCtx);
413       encodedResultWithHeader =
414           encodingCtx.getUncompressedBytesWithHeader();
415     } else {
416       HFileBlockDefaultEncodingContext defaultEncodingCtx =
417         new HFileBlockDefaultEncodingContext(algo, encoding, dummyHeader);
418       byte[] rawBufWithHeader =
419           new byte[rawBuf.array().length + headerLen];
420       System.arraycopy(rawBuf.array(), 0, rawBufWithHeader,
421           headerLen, rawBuf.array().length);
422       defaultEncodingCtx.compressAfterEncodingWithBlockType(rawBufWithHeader,
423           BlockType.DATA);
424       encodedResultWithHeader =
425         defaultEncodingCtx.getUncompressedBytesWithHeader();
426     }
427     final int encodedSize =
428         encodedResultWithHeader.length - headerLen;
429     if (encoder != null) {
430       // We need to account for the two-byte encoding algorithm ID that
431       // comes after the 24-byte block header but before encoded KVs.
432       headerLen += DataBlockEncoding.ID_SIZE;
433     }
434     byte[] encodedDataSection =
435         new byte[encodedResultWithHeader.length - headerLen];
436     System.arraycopy(encodedResultWithHeader, headerLen,
437         encodedDataSection, 0, encodedDataSection.length);
438     final ByteBuffer encodedBuf =
439         ByteBuffer.wrap(encodedDataSection);
440     encodedSizes.add(encodedSize);
441     encodedBlocks.add(encodedBuf);
442   }
443 
444   static void assertBuffersEqual(ByteBuffer expectedBuffer,
445       ByteBuffer actualBuffer, Compression.Algorithm compression,
446       DataBlockEncoding encoding, boolean pread) {
447     if (!actualBuffer.equals(expectedBuffer)) {
448       int prefix = 0;
449       int minLimit = Math.min(expectedBuffer.limit(), actualBuffer.limit());
450       while (prefix < minLimit &&
451           expectedBuffer.get(prefix) == actualBuffer.get(prefix)) {
452         prefix++;
453       }
454 
455       fail(String.format(
456           "Content mismath for compression %s, encoding %s, " +
457           "pread %s, commonPrefix %d, expected %s, got %s",
458           compression, encoding, pread, prefix,
459           nextBytesToStr(expectedBuffer, prefix),
460           nextBytesToStr(actualBuffer, prefix)));
461     }
462   }
463 
464   /**
465    * Convert a few next bytes in the given buffer at the given position to
466    * string. Used for error messages.
467    */
468   private static String nextBytesToStr(ByteBuffer buf, int pos) {
469     int maxBytes = buf.limit() - pos;
470     int numBytes = Math.min(16, maxBytes);
471     return Bytes.toStringBinary(buf.array(), buf.arrayOffset() + pos,
472         numBytes) + (numBytes < maxBytes ? "..." : "");
473   }
474 
475   @Test
476   public void testPreviousOffset() throws IOException {
477     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
478       for (boolean pread : BOOLEAN_VALUES) {
479         for (boolean cacheOnWrite : BOOLEAN_VALUES) {
480           Random rand = defaultRandom();
481           LOG.info("testPreviousOffset:Compression algorithm: " + algo +
482                    ", pread=" + pread +
483                    ", cacheOnWrite=" + cacheOnWrite);
484           Path path = new Path(TEST_UTIL.getDataTestDir(), "prev_offset");
485           List<Long> expectedOffsets = new ArrayList<Long>();
486           List<Long> expectedPrevOffsets = new ArrayList<Long>();
487           List<BlockType> expectedTypes = new ArrayList<BlockType>();
488           List<ByteBuffer> expectedContents = cacheOnWrite
489               ? new ArrayList<ByteBuffer>() : null;
490           long totalSize = writeBlocks(rand, algo, path, expectedOffsets,
491               expectedPrevOffsets, expectedTypes, expectedContents);
492 
493           FSDataInputStream is = fs.open(path);
494           HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo,
495               totalSize);
496           long curOffset = 0;
497           for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
498             if (!pread) {
499               assertEquals(is.getPos(), curOffset + (i == 0 ? 0 :
500                   HConstants.HFILEBLOCK_HEADER_SIZE));
501             }
502 
503             assertEquals(expectedOffsets.get(i).longValue(), curOffset);
504             if (detailedLogging) {
505               LOG.info("Reading block #" + i + " at offset " + curOffset);
506             }
507             HFileBlock b = hbr.readBlockData(curOffset, -1, -1, pread);
508             if (detailedLogging) {
509               LOG.info("Block #" + i + ": " + b);
510             }
511             assertEquals("Invalid block #" + i + "'s type:",
512                 expectedTypes.get(i), b.getBlockType());
513             assertEquals("Invalid previous block offset for block " + i
514                 + " of " + "type " + b.getBlockType() + ":",
515                 (long) expectedPrevOffsets.get(i), b.getPrevBlockOffset());
516             b.sanityCheck();
517             assertEquals(curOffset, b.getOffset());
518 
519             // Now re-load this block knowing the on-disk size. This tests a
520             // different branch in the loader.
521             HFileBlock b2 = hbr.readBlockData(curOffset,
522                 b.getOnDiskSizeWithHeader(), -1, pread);
523             b2.sanityCheck();
524 
525             assertEquals(b.getBlockType(), b2.getBlockType());
526             assertEquals(b.getOnDiskSizeWithoutHeader(),
527                 b2.getOnDiskSizeWithoutHeader());
528             assertEquals(b.getOnDiskSizeWithHeader(),
529                 b2.getOnDiskSizeWithHeader());
530             assertEquals(b.getUncompressedSizeWithoutHeader(),
531                 b2.getUncompressedSizeWithoutHeader());
532             assertEquals(b.getPrevBlockOffset(), b2.getPrevBlockOffset());
533             assertEquals(curOffset, b2.getOffset());
534             assertEquals(b.getBytesPerChecksum(), b2.getBytesPerChecksum());
535             assertEquals(b.getOnDiskDataSizeWithHeader(),
536                          b2.getOnDiskDataSizeWithHeader());
537             assertEquals(0, HFile.getChecksumFailuresCount());
538 
539             curOffset += b.getOnDiskSizeWithHeader();
540 
541             if (cacheOnWrite) {
542               // In the cache-on-write mode we store uncompressed bytes so we
543               // can compare them to what was read by the block reader.
544               // b's buffer has header + data + checksum while
545               // expectedContents have header + data only
546               ByteBuffer bufRead = b.getBufferWithHeader();
547               ByteBuffer bufExpected = expectedContents.get(i);
548               boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
549                   bufRead.arrayOffset(),
550                   bufRead.limit() - b.totalChecksumBytes(),
551                   bufExpected.array(), bufExpected.arrayOffset(),
552                   bufExpected.limit()) == 0;
553               String wrongBytesMsg = "";
554 
555               if (!bytesAreCorrect) {
556                 // Optimization: only construct an error message in case we
557                 // will need it.
558                 wrongBytesMsg = "Expected bytes in block #" + i + " (algo="
559                     + algo + ", pread=" + pread
560                     + ", cacheOnWrite=" + cacheOnWrite + "):\n";
561                 wrongBytesMsg += Bytes.toStringBinary(bufExpected.array(),
562                     bufExpected.arrayOffset(), Math.min(32,
563                         bufExpected.limit()))
564                     + ", actual:\n"
565                     + Bytes.toStringBinary(bufRead.array(),
566                         bufRead.arrayOffset(), Math.min(32, bufRead.limit()));
567                 if (detailedLogging) {
568                   LOG.warn("expected header" +
569                            HFileBlock.toStringHeader(bufExpected) +
570                            "\nfound    header" +
571                            HFileBlock.toStringHeader(bufRead));
572                   LOG.warn("bufread offset " + bufRead.arrayOffset() +
573                            " limit " + bufRead.limit() +
574                            " expected offset " + bufExpected.arrayOffset() +
575                            " limit " + bufExpected.limit());
576                   LOG.warn(wrongBytesMsg);
577                 }
578               }
579               assertTrue(wrongBytesMsg, bytesAreCorrect);
580             }
581           }
582 
583           assertEquals(curOffset, fs.getFileStatus(path).getLen());
584           is.close();
585         }
586       }
587     }
588   }
589 
590   private Random defaultRandom() {
591     return new Random(189237);
592   }
593 
594   private class BlockReaderThread implements Callable<Boolean> {
595     private final String clientId;
596     private final HFileBlock.FSReader hbr;
597     private final List<Long> offsets;
598     private final List<BlockType> types;
599     private final long fileSize;
600 
601     public BlockReaderThread(String clientId,
602         HFileBlock.FSReader hbr, List<Long> offsets, List<BlockType> types,
603         long fileSize) {
604       this.clientId = clientId;
605       this.offsets = offsets;
606       this.hbr = hbr;
607       this.types = types;
608       this.fileSize = fileSize;
609     }
610 
611     @Override
612     public Boolean call() throws Exception {
613       Random rand = new Random(clientId.hashCode());
614       long endTime = System.currentTimeMillis() + 10000;
615       int numBlocksRead = 0;
616       int numPositionalRead = 0;
617       int numWithOnDiskSize = 0;
618       while (System.currentTimeMillis() < endTime) {
619         int blockId = rand.nextInt(NUM_TEST_BLOCKS);
620         long offset = offsets.get(blockId);
621         boolean pread = rand.nextBoolean();
622         boolean withOnDiskSize = rand.nextBoolean();
623         long expectedSize =
624           (blockId == NUM_TEST_BLOCKS - 1 ? fileSize
625               : offsets.get(blockId + 1)) - offset;
626 
627         HFileBlock b;
628         try {
629           long onDiskSizeArg = withOnDiskSize ? expectedSize : -1;
630           b = hbr.readBlockData(offset, onDiskSizeArg, -1, pread);
631         } catch (IOException ex) {
632           LOG.error("Error in client " + clientId + " trying to read block at "
633               + offset + ", pread=" + pread + ", withOnDiskSize=" +
634               withOnDiskSize, ex);
635           return false;
636         }
637 
638         assertEquals(types.get(blockId), b.getBlockType());
639         assertEquals(expectedSize, b.getOnDiskSizeWithHeader());
640         assertEquals(offset, b.getOffset());
641 
642         ++numBlocksRead;
643         if (pread)
644           ++numPositionalRead;
645         if (withOnDiskSize)
646           ++numWithOnDiskSize;
647       }
648       LOG.info("Client " + clientId + " successfully read " + numBlocksRead +
649         " blocks (with pread: " + numPositionalRead + ", with onDiskSize " +
650         "specified: " + numWithOnDiskSize + ")");
651 
652       return true;
653     }
654 
655   }
656 
657   @Test
658   public void testConcurrentReading() throws Exception {
659     for (Compression.Algorithm compressAlgo : COMPRESSION_ALGORITHMS) {
660       Path path =
661           new Path(TEST_UTIL.getDataTestDir(), "concurrent_reading");
662       Random rand = defaultRandom();
663       List<Long> offsets = new ArrayList<Long>();
664       List<BlockType> types = new ArrayList<BlockType>();
665       writeBlocks(rand, compressAlgo, path, offsets, null, types, null);
666       FSDataInputStream is = fs.open(path);
667       long fileSize = fs.getFileStatus(path).getLen();
668       HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, compressAlgo,
669           fileSize);
670 
671       Executor exec = Executors.newFixedThreadPool(NUM_READER_THREADS);
672       ExecutorCompletionService<Boolean> ecs =
673           new ExecutorCompletionService<Boolean>(exec);
674 
675       for (int i = 0; i < NUM_READER_THREADS; ++i) {
676         ecs.submit(new BlockReaderThread("reader_" + (char) ('A' + i), hbr,
677             offsets, types, fileSize));
678       }
679 
680       for (int i = 0; i < NUM_READER_THREADS; ++i) {
681         Future<Boolean> result = ecs.take();
682         assertTrue(result.get());
683         if (detailedLogging) {
684           LOG.info(String.valueOf(i + 1)
685             + " reader threads finished successfully (algo=" + compressAlgo
686             + ")");
687         }
688       }
689 
690       is.close();
691     }
692   }
693 
694   private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
695       Path path, List<Long> expectedOffsets, List<Long> expectedPrevOffsets,
696       List<BlockType> expectedTypes, List<ByteBuffer> expectedContents
697   ) throws IOException {
698     boolean cacheOnWrite = expectedContents != null;
699     FSDataOutputStream os = fs.create(path);
700     HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null,
701         includesMemstoreTS, HFile.DEFAULT_CHECKSUM_TYPE,
702         HFile.DEFAULT_BYTES_PER_CHECKSUM);
703     Map<BlockType, Long> prevOffsetByType = new HashMap<BlockType, Long>();
704     long totalSize = 0;
705     for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
706       long pos = os.getPos();
707       int blockTypeOrdinal = rand.nextInt(BlockType.values().length);
708       if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) {
709         blockTypeOrdinal = BlockType.DATA.ordinal();
710       }
711       BlockType bt = BlockType.values()[blockTypeOrdinal];
712       DataOutputStream dos = hbw.startWriting(bt);
713       int size = rand.nextInt(500);
714       for (int j = 0; j < size; ++j) {
715         // This might compress well.
716         dos.writeShort(i + 1);
717         dos.writeInt(j + 1);
718       }
719 
720       if (expectedOffsets != null)
721         expectedOffsets.add(os.getPos());
722 
723       if (expectedPrevOffsets != null) {
724         Long prevOffset = prevOffsetByType.get(bt);
725         expectedPrevOffsets.add(prevOffset != null ? prevOffset : -1);
726         prevOffsetByType.put(bt, os.getPos());
727       }
728 
729       expectedTypes.add(bt);
730 
731       hbw.writeHeaderAndData(os);
732       totalSize += hbw.getOnDiskSizeWithHeader();
733 
734       if (cacheOnWrite)
735         expectedContents.add(hbw.getUncompressedBufferWithHeader());
736 
737       if (detailedLogging) {
738         LOG.info("Written block #" + i + " of type " + bt
739             + ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader()
740             + " at offset " + pos);
741       }
742     }
743     os.close();
744     LOG.info("Created a temporary file at " + path + ", "
745         + fs.getFileStatus(path).getLen() + " byte, compression=" +
746         compressAlgo);
747     return totalSize;
748   }
749 
750   @Test
751   public void testBlockHeapSize() {
752     if (ClassSize.is32BitJVM()) {
753       assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 64);
754     } else {
755       assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 80);
756     }
757 
758     for (int size : new int[] { 100, 256, 12345 }) {
759       byte[] byteArr = new byte[HConstants.HFILEBLOCK_HEADER_SIZE + size];
760       ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
761       HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
762           HFileBlock.FILL_HEADER, -1, includesMemstoreTS,
763           HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(),
764           0);
765       long byteBufferExpectedSize =
766           ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
767               + HConstants.HFILEBLOCK_HEADER_SIZE + size);
768       long hfileBlockExpectedSize =
769           ClassSize.align(ClassSize.estimateBase(HFileBlock.class, true));
770       long expected = hfileBlockExpectedSize + byteBufferExpectedSize;
771       assertEquals("Block data size: " + size + ", byte buffer expected " +
772           "size: " + byteBufferExpectedSize + ", HFileBlock class expected " +
773           "size: " + hfileBlockExpectedSize + ";", expected,
774           block.heapSize());
775     }
776   }
777 
778 
779 }
780