1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import static org.junit.Assert.*;
23  
24  import java.io.ByteArrayOutputStream;
25  import java.io.DataOutputStream;
26  import java.io.IOException;
27  import java.io.OutputStream;
28  import java.nio.ByteBuffer;
29  import java.util.ArrayList;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.HashMap;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Random;
36  import java.util.concurrent.Callable;
37  import java.util.concurrent.Executor;
38  import java.util.concurrent.ExecutorCompletionService;
39  import java.util.concurrent.Executors;
40  import java.util.concurrent.Future;
41  
42  import org.apache.commons.logging.Log;
43  import org.apache.commons.logging.LogFactory;
44  import org.apache.hadoop.fs.FSDataInputStream;
45  import org.apache.hadoop.fs.FSDataOutputStream;
46  import org.apache.hadoop.fs.FileSystem;
47  import org.apache.hadoop.fs.Path;
48  import org.apache.hadoop.hbase.HBaseTestingUtility;
49  import org.apache.hadoop.hbase.MediumTests;
50  import org.apache.hadoop.hbase.KeyValue;
51  import org.apache.hadoop.hbase.fs.HFileSystem;
52  import org.apache.hadoop.hbase.io.DoubleOutputStream;
53  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
54  import org.apache.hadoop.hbase.util.Bytes;
55  import org.apache.hadoop.hbase.util.ChecksumType;
56  import org.apache.hadoop.hbase.util.ClassSize;
57  import org.apache.hadoop.io.WritableUtils;
58  import org.apache.hadoop.io.compress.CompressionOutputStream;
59  import org.apache.hadoop.io.compress.Compressor;
60  
61  import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.*;
62  import org.junit.Before;
63  import org.junit.Test;
64  import org.junit.experimental.categories.Category;
65  import org.junit.runner.RunWith;
66  import org.junit.runners.Parameterized;
67  import org.junit.runners.Parameterized.Parameters;
68  
69  @Category(MediumTests.class)
70  @RunWith(Parameterized.class)
71  public class TestHFileBlock {
72    // change this value to activate more logs
73    private static final boolean detailedLogging = false;
74    private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true };
75  
76    private static final Log LOG = LogFactory.getLog(TestHFileBlock.class);
77  
78    static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
79        NONE, GZ };
80  
81    private static final int NUM_TEST_BLOCKS = 1000;
82    private static final int NUM_READER_THREADS = 26;
83  
84    // Used to generate KeyValues
85    private static int NUM_KEYVALUES = 50;
86    private static int FIELD_LENGTH = 10;
87    private static float CHANCE_TO_REPEAT = 0.6f;
88  
89    private static final HBaseTestingUtility TEST_UTIL =
90      new HBaseTestingUtility();
91    private FileSystem fs;
92    private int uncompressedSizeV1;
93  
94    private final boolean includesMemstoreTS;
95  
96    public TestHFileBlock(boolean includesMemstoreTS) {
97      this.includesMemstoreTS = includesMemstoreTS;
98    }
99  
100   @Parameters
101   public static Collection<Object[]> parameters() {
102     return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
103   }
104 
105   @Before
106   public void setUp() throws IOException {
107     fs = HFileSystem.get(TEST_UTIL.getConfiguration());
108   }
109 
110   static void writeTestBlockContents(DataOutputStream dos) throws IOException {
111     // This compresses really well.
112     for (int i = 0; i < 1000; ++i)
113       dos.writeInt(i / 100);
114   }
115 
116  static int writeTestKeyValues(OutputStream dos, int seed, boolean includesMemstoreTS)
117       throws IOException {
118     List<KeyValue> keyValues = new ArrayList<KeyValue>();
119     Random randomizer = new Random(42l + seed); // just any fixed number
120 
121     // generate keyValues
122     for (int i = 0; i < NUM_KEYVALUES; ++i) {
123       byte[] row;
124       long timestamp;
125       byte[] family;
126       byte[] qualifier;
127       byte[] value;
128 
129       // generate it or repeat, it should compress well
130       if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
131         row = keyValues.get(randomizer.nextInt(keyValues.size())).getRow();
132       } else {
133         row = new byte[FIELD_LENGTH];
134         randomizer.nextBytes(row);
135       }
136       if (0 == i) {
137         family = new byte[FIELD_LENGTH];
138         randomizer.nextBytes(family);
139       } else {
140         family = keyValues.get(0).getFamily();
141       }
142       if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
143         qualifier = keyValues.get(
144             randomizer.nextInt(keyValues.size())).getQualifier();
145       } else {
146         qualifier = new byte[FIELD_LENGTH];
147         randomizer.nextBytes(qualifier);
148       }
149       if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
150         value = keyValues.get(randomizer.nextInt(keyValues.size())).getValue();
151       } else {
152         value = new byte[FIELD_LENGTH];
153         randomizer.nextBytes(value);
154       }
155       if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
156         timestamp = keyValues.get(
157             randomizer.nextInt(keyValues.size())).getTimestamp();
158       } else {
159         timestamp = randomizer.nextLong();
160       }
161 
162       keyValues.add(new KeyValue(row, family, qualifier, timestamp, value));
163     }
164 
165     // sort it and write to stream
166     int totalSize = 0;
167     Collections.sort(keyValues, KeyValue.COMPARATOR);
168     DataOutputStream dataOutputStream = new DataOutputStream(dos);
169     for (KeyValue kv : keyValues) {
170       totalSize += kv.getLength();
171       dataOutputStream.write(kv.getBuffer(), kv.getOffset(), kv.getLength());
172       if (includesMemstoreTS) {
173         long memstoreTS = randomizer.nextLong();
174         WritableUtils.writeVLong(dataOutputStream, memstoreTS);
175         totalSize += WritableUtils.getVIntSize(memstoreTS);
176       }
177     }
178 
179     return totalSize;
180   }
181 
182   public byte[] createTestV1Block(Compression.Algorithm algo)
183       throws IOException {
184     Compressor compressor = algo.getCompressor();
185     ByteArrayOutputStream baos = new ByteArrayOutputStream();
186     OutputStream os = algo.createCompressionStream(baos, compressor, 0);
187     DataOutputStream dos = new DataOutputStream(os);
188     BlockType.META.write(dos); // Let's make this a meta block.
189     writeTestBlockContents(dos);
190     uncompressedSizeV1 = dos.size();
191     dos.flush();
192     algo.returnCompressor(compressor);
193     return baos.toByteArray();
194   }
195 
196   static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo,
197       boolean includesMemstoreTS) throws IOException {
198     final BlockType blockType = BlockType.DATA;
199     HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
200         includesMemstoreTS, HFileReaderV2.MAX_MINOR_VERSION,
201         HFile.DEFAULT_CHECKSUM_TYPE,
202         HFile.DEFAULT_BYTES_PER_CHECKSUM);
203     DataOutputStream dos = hbw.startWriting(blockType);
204     writeTestBlockContents(dos);
205     dos.flush();
206     byte[] headerAndData = hbw.getHeaderAndDataForTest();
207     assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader());
208     hbw.releaseCompressor();
209     return hbw;
210   }
211 
212   public String createTestBlockStr(Compression.Algorithm algo,
213       int correctLength) throws IOException {
214     HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS);
215     byte[] testV2Block = hbw.getHeaderAndDataForTest();
216     int osOffset = HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + 9;
217     if (testV2Block.length == correctLength) {
218       // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
219       // variations across operating systems.
220       // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format.
221       // We only make this change when the compressed block length matches.
222       // Otherwise, there are obviously other inconsistencies.
223       testV2Block[osOffset] = 3;
224     }
225     return Bytes.toStringBinary(testV2Block);
226   }
227 
228   @Test
229   public void testNoCompression() throws IOException {
230     assertEquals(4000, createTestV2Block(NONE, includesMemstoreTS).
231                  getBlockForCaching().getUncompressedSizeWithoutHeader());
232   }
233 
234   @Test
235   public void testGzipCompression() throws IOException {
236     final String correctTestBlockStr =
237         "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF"
238             + "\\xFF\\xFF\\xFF\\xFF"
239             + "\\x01\\x00\\x00@\\x00\\x00\\x00\\x00["
240             // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html
241             + "\\x1F\\x8B"  // gzip magic signature
242             + "\\x08"  // Compression method: 8 = "deflate"
243             + "\\x00"  // Flags
244             + "\\x00\\x00\\x00\\x00"  // mtime
245             + "\\x00"  // XFL (extra flags)
246             // OS (0 = FAT filesystems, 3 = Unix). However, this field
247             // sometimes gets set to 0 on Linux and Mac, so we reset it to 3.
248             // This appears to be a difference caused by the availability
249             // (and use) of the native GZ codec.
250             + "\\x03"
251             + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa"
252             + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c"
253             + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00"
254             + "\\x00\\x00\\x00\\x00"; //  4 byte checksum (ignored)
255     final int correctGzipBlockLength = 95;
256     final String testBlockStr = createTestBlockStr(GZ, correctGzipBlockLength);
257     // We ignore the block checksum because createTestBlockStr can change the
258     // gzip header after the block is produced
259     assertEquals(correctTestBlockStr.substring(0, correctGzipBlockLength - 4),
260       testBlockStr.substring(0, correctGzipBlockLength - 4));
261   }
262 
263   @Test
264   public void testReaderV1() throws IOException {
265     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
266       for (boolean pread : new boolean[] { false, true }) {
267         byte[] block = createTestV1Block(algo);
268         Path path = new Path(TEST_UTIL.getDataTestDir(),
269           "blocks_v1_"+ algo);
270         LOG.info("Creating temporary file at " + path);
271         FSDataOutputStream os = fs.create(path);
272         int totalSize = 0;
273         int numBlocks = 50;
274         for (int i = 0; i < numBlocks; ++i) {
275           os.write(block);
276           totalSize += block.length;
277         }
278         os.close();
279 
280         FSDataInputStream is = fs.open(path);
281         HFileBlock.FSReader hbr = new HFileBlock.FSReaderV1(is, algo,
282             totalSize);
283         HFileBlock b;
284         int numBlocksRead = 0;
285         long pos = 0;
286         while (pos < totalSize) {
287           b = hbr.readBlockData(pos, block.length, uncompressedSizeV1, pread);
288           b.sanityCheck();
289           pos += block.length;
290           numBlocksRead++;
291         }
292         assertEquals(numBlocks, numBlocksRead);
293         is.close();
294       }
295     }
296   }
297 
298   @Test
299   public void testReaderV2() throws IOException {
300     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
301       for (boolean pread : new boolean[] { false, true }) {
302           LOG.info("testReaderV2: Compression algorithm: " + algo + 
303                    ", pread=" + pread);
304         Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
305             + algo);
306         FSDataOutputStream os = fs.create(path);
307         HFileBlock.Writer hbw = new HFileBlock.Writer(algo, null,
308             includesMemstoreTS,
309             HFileReaderV2.MAX_MINOR_VERSION,
310             HFile.DEFAULT_CHECKSUM_TYPE,
311             HFile.DEFAULT_BYTES_PER_CHECKSUM);
312         long totalSize = 0;
313         for (int blockId = 0; blockId < 2; ++blockId) {
314           DataOutputStream dos = hbw.startWriting(BlockType.DATA);
315           for (int i = 0; i < 1234; ++i)
316             dos.writeInt(i);
317           hbw.writeHeaderAndData(os);
318           totalSize += hbw.getOnDiskSizeWithHeader();
319         }
320         os.close();
321 
322         FSDataInputStream is = fs.open(path);
323         HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo,
324             totalSize);
325         HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
326         is.close();
327         assertEquals(0, HFile.getChecksumFailuresCount());
328 
329         b.sanityCheck();
330         assertEquals(4936, b.getUncompressedSizeWithoutHeader());
331         assertEquals(algo == GZ ? 2173 : 4936, 
332                      b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
333         String blockStr = b.toString();
334 
335         if (algo == GZ) {
336           is = fs.open(path);
337           hbr = new HFileBlock.FSReaderV2(is, algo, totalSize);
338           b = hbr.readBlockData(0, 2173 + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS +
339                                 b.totalChecksumBytes(), -1, pread);
340           assertEquals(blockStr, b.toString());
341           int wrongCompressedSize = 2172;
342           try {
343             b = hbr.readBlockData(0, wrongCompressedSize
344                 + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS, -1, pread);
345             fail("Exception expected");
346           } catch (IOException ex) {
347             String expectedPrefix = "On-disk size without header provided is "
348                 + wrongCompressedSize + ", but block header contains "
349                 + b.getOnDiskSizeWithoutHeader() + ".";
350             assertTrue("Invalid exception message: '" + ex.getMessage()
351                 + "'.\nMessage is expected to start with: '" + expectedPrefix
352                 + "'", ex.getMessage().startsWith(expectedPrefix));
353           }
354           is.close();
355         }
356       }
357     }
358   }
359 
360   /**
361    * Test encoding/decoding data blocks.
362    * @throws IOException a bug or a problem with temporary files.
363    */
364   @Test
365   public void testDataBlockEncoding() throws IOException {
366     final int numBlocks = 5;
367     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
368       for (boolean pread : new boolean[] { false, true }) {
369         for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
370           Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_"
371               + algo + "_" + encoding.toString());
372           FSDataOutputStream os = fs.create(path);
373           HFileDataBlockEncoder dataBlockEncoder =
374               new HFileDataBlockEncoderImpl(encoding);
375           HFileBlock.Writer hbw = new HFileBlock.Writer(algo, dataBlockEncoder,
376               includesMemstoreTS,
377               HFileReaderV2.MAX_MINOR_VERSION,
378               HFile.DEFAULT_CHECKSUM_TYPE,
379               HFile.DEFAULT_BYTES_PER_CHECKSUM);
380           long totalSize = 0;
381           final List<Integer> encodedSizes = new ArrayList<Integer>();
382           final List<ByteBuffer> encodedBlocks = new ArrayList<ByteBuffer>();
383           for (int blockId = 0; blockId < numBlocks; ++blockId) {
384             DataOutputStream dos = hbw.startWriting(BlockType.DATA);
385             writeEncodedBlock(encoding, dos, encodedSizes, encodedBlocks,
386                 blockId, includesMemstoreTS);
387 
388             hbw.writeHeaderAndData(os);
389             totalSize += hbw.getOnDiskSizeWithHeader();
390           }
391           os.close();
392 
393           FSDataInputStream is = fs.open(path);
394           HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, algo,
395               totalSize);
396           hbr.setDataBlockEncoder(dataBlockEncoder);
397           hbr.setIncludesMemstoreTS(includesMemstoreTS);
398 
399           HFileBlock b;
400           int pos = 0;
401           for (int blockId = 0; blockId < numBlocks; ++blockId) {
402             b = hbr.readBlockData(pos, -1, -1, pread);
403             assertEquals(0, HFile.getChecksumFailuresCount());
404             b.sanityCheck();
405             pos += b.getOnDiskSizeWithHeader();
406 
407             assertEquals((int) encodedSizes.get(blockId),
408                 b.getUncompressedSizeWithoutHeader());
409             ByteBuffer actualBuffer = b.getBufferWithoutHeader();
410             if (encoding != DataBlockEncoding.NONE) {
411               // We expect a two-byte big-endian encoding id.
412               assertEquals(0, actualBuffer.get(0));
413               assertEquals(encoding.getId(), actualBuffer.get(1));
414               actualBuffer.position(2);
415               actualBuffer = actualBuffer.slice();
416             }
417 
418             ByteBuffer expectedBuffer = encodedBlocks.get(blockId);
419             expectedBuffer.rewind();
420 
421             // test if content matches, produce nice message
422             assertBuffersEqual(expectedBuffer, actualBuffer, algo, encoding,
423                 pread);
424           }
425           is.close();
426         }
427       }
428     }
429   }
430 
431   static void writeEncodedBlock(DataBlockEncoding encoding,
432       DataOutputStream dos, final List<Integer> encodedSizes,
433       final List<ByteBuffer> encodedBlocks, int blockId, 
434       boolean includesMemstoreTS) throws IOException {
435     ByteArrayOutputStream baos = new ByteArrayOutputStream();
436     DoubleOutputStream doubleOutputStream =
437         new DoubleOutputStream(dos, baos);
438 
439     final int rawBlockSize = writeTestKeyValues(doubleOutputStream,
440         blockId, includesMemstoreTS);
441 
442     ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
443     rawBuf.rewind();
444 
445     final int encodedSize;
446     final ByteBuffer encodedBuf;
447     if (encoding == DataBlockEncoding.NONE) {
448       encodedSize = rawBlockSize;
449       encodedBuf = rawBuf;
450     } else {
451       ByteArrayOutputStream encodedOut = new ByteArrayOutputStream();
452       encoding.getEncoder().compressKeyValues(
453           new DataOutputStream(encodedOut),
454           rawBuf.duplicate(), includesMemstoreTS);
455       // We need to account for the two-byte encoding algorithm ID that
456       // comes after the 24-byte block header but before encoded KVs.
457       encodedSize = encodedOut.size() + DataBlockEncoding.ID_SIZE;
458       encodedBuf = ByteBuffer.wrap(encodedOut.toByteArray());
459     }
460     encodedSizes.add(encodedSize);
461     encodedBlocks.add(encodedBuf);
462   }
463 
464   static void assertBuffersEqual(ByteBuffer expectedBuffer,
465       ByteBuffer actualBuffer, Compression.Algorithm compression,
466       DataBlockEncoding encoding, boolean pread) {
467     if (!actualBuffer.equals(expectedBuffer)) {
468       int prefix = 0;
469       int minLimit = Math.min(expectedBuffer.limit(), actualBuffer.limit());
470       while (prefix < minLimit &&
471           expectedBuffer.get(prefix) == actualBuffer.get(prefix)) {
472         prefix++;
473       }
474 
475       fail(String.format(
476           "Content mismath for compression %s, encoding %s, " +
477           "pread %s, commonPrefix %d, expected %s, got %s",
478           compression, encoding, pread, prefix,
479           nextBytesToStr(expectedBuffer, prefix),
480           nextBytesToStr(actualBuffer, prefix)));
481     }
482   }
483 
484   /**
485    * Convert a few next bytes in the given buffer at the given position to
486    * string. Used for error messages.
487    */
488   private static String nextBytesToStr(ByteBuffer buf, int pos) {
489     int maxBytes = buf.limit() - pos;
490     int numBytes = Math.min(16, maxBytes);
491     return Bytes.toStringBinary(buf.array(), buf.arrayOffset() + pos,
492         numBytes) + (numBytes < maxBytes ? "..." : "");
493   }
494 
495   @Test
496   public void testPreviousOffset() throws IOException {
497     for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
498       for (boolean pread : BOOLEAN_VALUES) {
499         for (boolean cacheOnWrite : BOOLEAN_VALUES) {
500           Random rand = defaultRandom();
501           LOG.info("testPreviousOffset:Compression algorithm: " + algo + 
502                    ", pread=" + pread +
503                    ", cacheOnWrite=" + cacheOnWrite);
504           Path path = new Path(TEST_UTIL.getDataTestDir(), "prev_offset");
505           List<Long> expectedOffsets = new ArrayList<Long>();
506           List<Long> expectedPrevOffsets = new ArrayList<Long>();
507           List<BlockType> expectedTypes = new ArrayList<BlockType>();
508           List<ByteBuffer> expectedContents = cacheOnWrite
509               ? new ArrayList<ByteBuffer>() : null;
510           long totalSize = writeBlocks(rand, algo, path, expectedOffsets,
511               expectedPrevOffsets, expectedTypes, expectedContents);
512 
513           FSDataInputStream is = fs.open(path);
514           HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, algo,
515               totalSize);
516           long curOffset = 0;
517           for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
518             if (!pread) {
519               assertEquals(is.getPos(), curOffset + (i == 0 ? 0 :
520                   HFileBlock.HEADER_SIZE_WITH_CHECKSUMS));
521             }
522 
523             assertEquals(expectedOffsets.get(i).longValue(), curOffset);
524             if (detailedLogging) {
525               LOG.info("Reading block #" + i + " at offset " + curOffset);
526             }
527             HFileBlock b = hbr.readBlockData(curOffset, -1, -1, pread);
528             if (detailedLogging) {
529               LOG.info("Block #" + i + ": " + b);
530             }
531             assertEquals("Invalid block #" + i + "'s type:",
532                 expectedTypes.get(i), b.getBlockType());
533             assertEquals("Invalid previous block offset for block " + i
534                 + " of " + "type " + b.getBlockType() + ":",
535                 (long) expectedPrevOffsets.get(i), b.getPrevBlockOffset());
536             b.sanityCheck();
537             assertEquals(curOffset, b.getOffset());
538 
539             // Now re-load this block knowing the on-disk size. This tests a
540             // different branch in the loader.
541             HFileBlock b2 = hbr.readBlockData(curOffset,
542                 b.getOnDiskSizeWithHeader(), -1, pread);
543             b2.sanityCheck();
544 
545             assertEquals(b.getBlockType(), b2.getBlockType());
546             assertEquals(b.getOnDiskSizeWithoutHeader(),
547                 b2.getOnDiskSizeWithoutHeader());
548             assertEquals(b.getOnDiskSizeWithHeader(),
549                 b2.getOnDiskSizeWithHeader());
550             assertEquals(b.getUncompressedSizeWithoutHeader(),
551                 b2.getUncompressedSizeWithoutHeader());
552             assertEquals(b.getPrevBlockOffset(), b2.getPrevBlockOffset());
553             assertEquals(curOffset, b2.getOffset());
554             assertEquals(b.getBytesPerChecksum(), b2.getBytesPerChecksum());
555             assertEquals(b.getOnDiskDataSizeWithHeader(), 
556                          b2.getOnDiskDataSizeWithHeader());
557             assertEquals(0, HFile.getChecksumFailuresCount());
558 
559             curOffset += b.getOnDiskSizeWithHeader();
560 
561             if (cacheOnWrite) {
562               // In the cache-on-write mode we store uncompressed bytes so we
563               // can compare them to what was read by the block reader.
564               // b's buffer has header + data + checksum while 
565               // expectedContents have header + data only
566               ByteBuffer bufRead = b.getBufferWithHeader();
567               ByteBuffer bufExpected = expectedContents.get(i);
568               boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
569                   bufRead.arrayOffset(), 
570                   bufRead.limit() - b.totalChecksumBytes(),
571                   bufExpected.array(), bufExpected.arrayOffset(),
572                   bufExpected.limit()) == 0;
573               String wrongBytesMsg = "";
574 
575               if (!bytesAreCorrect) {
576                 // Optimization: only construct an error message in case we
577                 // will need it.
578                 wrongBytesMsg = "Expected bytes in block #" + i + " (algo="
579                     + algo + ", pread=" + pread
580                     + ", cacheOnWrite=" + cacheOnWrite + "):\n";
581                 wrongBytesMsg += Bytes.toStringBinary(bufExpected.array(),
582                     bufExpected.arrayOffset(), Math.min(32,
583                         bufExpected.limit()))
584                     + ", actual:\n"
585                     + Bytes.toStringBinary(bufRead.array(),
586                         bufRead.arrayOffset(), Math.min(32, bufRead.limit()));
587                 if (detailedLogging) {
588                   LOG.warn("expected header" + 
589                            HFileBlock.toStringHeader(bufExpected) +
590                            "\nfound    header" + 
591                            HFileBlock.toStringHeader(bufRead));
592                   LOG.warn("bufread offset " + bufRead.arrayOffset() +
593                            " limit " + bufRead.limit() +
594                            " expected offset " + bufExpected.arrayOffset() +
595                            " limit " + bufExpected.limit());
596                   LOG.warn(wrongBytesMsg);
597                 }
598               }
599               assertTrue(wrongBytesMsg, bytesAreCorrect);
600             }
601           }
602 
603           assertEquals(curOffset, fs.getFileStatus(path).getLen());
604           is.close();
605         }
606       }
607     }
608   }
609 
610   private Random defaultRandom() {
611     return new Random(189237);
612   }
613 
614   private class BlockReaderThread implements Callable<Boolean> {
615     private final String clientId;
616     private final HFileBlock.FSReader hbr;
617     private final List<Long> offsets;
618     private final List<BlockType> types;
619     private final long fileSize;
620 
621     public BlockReaderThread(String clientId,
622         HFileBlock.FSReader hbr, List<Long> offsets, List<BlockType> types,
623         long fileSize) {
624       this.clientId = clientId;
625       this.offsets = offsets;
626       this.hbr = hbr;
627       this.types = types;
628       this.fileSize = fileSize;
629     }
630 
631     @Override
632     public Boolean call() throws Exception {
633       Random rand = new Random(clientId.hashCode());
634       long endTime = System.currentTimeMillis() + 10000;
635       int numBlocksRead = 0;
636       int numPositionalRead = 0;
637       int numWithOnDiskSize = 0;
638       while (System.currentTimeMillis() < endTime) {
639         int blockId = rand.nextInt(NUM_TEST_BLOCKS);
640         long offset = offsets.get(blockId);
641         boolean pread = rand.nextBoolean();
642         boolean withOnDiskSize = rand.nextBoolean();
643         long expectedSize =
644           (blockId == NUM_TEST_BLOCKS - 1 ? fileSize
645               : offsets.get(blockId + 1)) - offset;
646 
647         HFileBlock b;
648         try {
649           long onDiskSizeArg = withOnDiskSize ? expectedSize : -1;
650           b = hbr.readBlockData(offset, onDiskSizeArg, -1, pread);
651         } catch (IOException ex) {
652           LOG.error("Error in client " + clientId + " trying to read block at "
653               + offset + ", pread=" + pread + ", withOnDiskSize=" +
654               withOnDiskSize, ex);
655           return false;
656         }
657 
658         assertEquals(types.get(blockId), b.getBlockType());
659         assertEquals(expectedSize, b.getOnDiskSizeWithHeader());
660         assertEquals(offset, b.getOffset());
661 
662         ++numBlocksRead;
663         if (pread)
664           ++numPositionalRead;
665         if (withOnDiskSize)
666           ++numWithOnDiskSize;
667       }
668       LOG.info("Client " + clientId + " successfully read " + numBlocksRead +
669         " blocks (with pread: " + numPositionalRead + ", with onDiskSize " +
670         "specified: " + numWithOnDiskSize + ")");
671 
672       return true;
673     }
674 
675   }
676 
677   @Test
678   public void testConcurrentReading() throws Exception {
679     for (Compression.Algorithm compressAlgo : COMPRESSION_ALGORITHMS) {
680       Path path =
681           new Path(TEST_UTIL.getDataTestDir(), "concurrent_reading");
682       Random rand = defaultRandom();
683       List<Long> offsets = new ArrayList<Long>();
684       List<BlockType> types = new ArrayList<BlockType>();
685       writeBlocks(rand, compressAlgo, path, offsets, null, types, null);
686       FSDataInputStream is = fs.open(path);
687       long fileSize = fs.getFileStatus(path).getLen();
688       HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, compressAlgo,
689           fileSize);
690 
691       Executor exec = Executors.newFixedThreadPool(NUM_READER_THREADS);
692       ExecutorCompletionService<Boolean> ecs =
693           new ExecutorCompletionService<Boolean>(exec);
694 
695       for (int i = 0; i < NUM_READER_THREADS; ++i) {
696         ecs.submit(new BlockReaderThread("reader_" + (char) ('A' + i), hbr,
697             offsets, types, fileSize));
698       }
699 
700       for (int i = 0; i < NUM_READER_THREADS; ++i) {
701         Future<Boolean> result = ecs.take();
702         assertTrue(result.get());
703         if (detailedLogging) {
704           LOG.info(String.valueOf(i + 1)
705             + " reader threads finished successfully (algo=" + compressAlgo
706             + ")");
707         }
708       }
709 
710       is.close();
711     }
712   }
713 
714   private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
715       Path path, List<Long> expectedOffsets, List<Long> expectedPrevOffsets,
716       List<BlockType> expectedTypes, List<ByteBuffer> expectedContents
717   ) throws IOException {
718     boolean cacheOnWrite = expectedContents != null;
719     FSDataOutputStream os = fs.create(path);
720     HFileBlock.Writer hbw = new HFileBlock.Writer(compressAlgo, null,
721         includesMemstoreTS,
722         HFileReaderV2.MAX_MINOR_VERSION,
723         HFile.DEFAULT_CHECKSUM_TYPE,
724         HFile.DEFAULT_BYTES_PER_CHECKSUM);
725     Map<BlockType, Long> prevOffsetByType = new HashMap<BlockType, Long>();
726     long totalSize = 0;
727     for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
728       long pos = os.getPos();
729       int blockTypeOrdinal = rand.nextInt(BlockType.values().length);
730       if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) {
731         blockTypeOrdinal = BlockType.DATA.ordinal();
732       }
733       BlockType bt = BlockType.values()[blockTypeOrdinal];
734       DataOutputStream dos = hbw.startWriting(bt);
735       int size = rand.nextInt(500);
736       for (int j = 0; j < size; ++j) {
737         // This might compress well.
738         dos.writeShort(i + 1);
739         dos.writeInt(j + 1);
740       }
741 
742       if (expectedOffsets != null)
743         expectedOffsets.add(os.getPos());
744 
745       if (expectedPrevOffsets != null) {
746         Long prevOffset = prevOffsetByType.get(bt);
747         expectedPrevOffsets.add(prevOffset != null ? prevOffset : -1);
748         prevOffsetByType.put(bt, os.getPos());
749       }
750 
751       expectedTypes.add(bt);
752 
753       hbw.writeHeaderAndData(os);
754       totalSize += hbw.getOnDiskSizeWithHeader();
755 
756       if (cacheOnWrite)
757         expectedContents.add(hbw.getUncompressedBufferWithHeader());
758 
759       if (detailedLogging) {
760         LOG.info("Written block #" + i + " of type " + bt
761             + ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader()
762             + " at offset " + pos);
763       }
764     }
765     os.close();
766     LOG.info("Created a temporary file at " + path + ", "
767         + fs.getFileStatus(path).getLen() + " byte, compression=" +
768         compressAlgo);
769     return totalSize;
770   }
771 
772   @Test
773   public void testBlockHeapSize() {
774     if (ClassSize.is32BitJVM()) {
775       assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 64);
776     } else {
777       assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 80);
778     }
779 
780     for (int size : new int[] { 100, 256, 12345 }) {
781       byte[] byteArr = new byte[HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + size];
782       ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
783       HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
784           HFileBlock.FILL_HEADER, -1, includesMemstoreTS, 
785           HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(),
786           0);
787       long byteBufferExpectedSize =
788           ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
789               + HFileBlock.HEADER_SIZE_WITH_CHECKSUMS + size);
790       long hfileBlockExpectedSize =
791           ClassSize.align(ClassSize.estimateBase(HFileBlock.class, true));
792       long expected = hfileBlockExpectedSize + byteBufferExpectedSize;
793       assertEquals("Block data size: " + size + ", byte buffer expected " +
794           "size: " + byteBufferExpectedSize + ", HFileBlock class expected " +
795           "size: " + hfileBlockExpectedSize + ";", expected,
796           block.heapSize());
797     }
798   }
799 
800 
801   @org.junit.Rule
802   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
803     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
804 }
805