View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.hfile;
18  
19  import static org.junit.Assert.assertEquals;
20  import static org.junit.Assert.assertTrue;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.ArrayList;
25  import java.util.Collection;
26  import java.util.List;
27  
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.SmallTests;
32  import org.apache.hadoop.hbase.io.HeapSize;
33  import org.apache.hadoop.hbase.io.compress.Compression;
34  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
35  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
36  import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
37  import org.apache.hadoop.hbase.util.ChecksumType;
38  import org.apache.hadoop.hbase.util.test.RedundantKVGenerator;
39  import org.junit.Before;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  import org.junit.runner.RunWith;
43  import org.junit.runners.Parameterized;
44  import org.junit.runners.Parameterized.Parameters;
45  
46  @RunWith(Parameterized.class)
47  @Category(SmallTests.class)
48  public class TestHFileDataBlockEncoder {
49    private Configuration conf;
50    private final HBaseTestingUtility TEST_UTIL =
51        new HBaseTestingUtility();
52    private HFileDataBlockEncoderImpl blockEncoder;
53    private RedundantKVGenerator generator = new RedundantKVGenerator();
54    private boolean includesMemstoreTS;
55  
56    /**
57     * Create test for given data block encoding configuration.
58     * @param blockEncoder What kind of encoding policy will be used.
59     */
60    public TestHFileDataBlockEncoder(HFileDataBlockEncoderImpl blockEncoder,
61        boolean includesMemstoreTS) {
62      this.blockEncoder = blockEncoder;
63      this.includesMemstoreTS = includesMemstoreTS;
64      System.err.println("On-disk encoding: " + blockEncoder.getEncodingOnDisk()
65          + ", in-cache encoding: " + blockEncoder.getEncodingInCache()
66          + ", includesMemstoreTS: " + includesMemstoreTS);
67    }
68  
69    /**
70     * Preparation before JUnit test.
71     */
72    @Before
73    public void setUp() {
74      conf = TEST_UTIL.getConfiguration();
75    }
76  
77    /**
78     * Test putting and taking out blocks into cache with different
79     * encoding options.
80     */
81    @Test
82    public void testEncodingWithCache() {
83      HFileBlock block = getSampleHFileBlock();
84      LruBlockCache blockCache =
85          new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
86      HFileBlock cacheBlock = blockEncoder.diskToCacheFormat(block, false);
87      BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
88      blockCache.cacheBlock(cacheKey, cacheBlock);
89  
90      HeapSize heapSize = blockCache.getBlock(cacheKey, false, false);
91      assertTrue(heapSize instanceof HFileBlock);
92  
93      HFileBlock returnedBlock = (HFileBlock) heapSize;;
94  
95      if (blockEncoder.getEncodingInCache() ==
96          DataBlockEncoding.NONE) {
97        assertEquals(block.getBufferWithHeader(),
98            returnedBlock.getBufferWithHeader());
99      } else {
100       if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
101         System.out.println(blockEncoder);
102       }
103       assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
104     }
105   }
106 
107   /** Test for HBASE-5746. */
108   @Test
109   public void testHeaderSizeInCacheWithoutChecksum() throws Exception {
110     int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
111     // Create some KVs and create the block with old-style header.
112     ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
113         generator.generateTestKeyValues(60), includesMemstoreTS);
114     int size = keyValues.limit();
115     ByteBuffer buf = ByteBuffer.allocate(size + headerSize);
116     buf.position(headerSize);
117     keyValues.rewind();
118     buf.put(keyValues);
119     HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
120         HFileBlock.FILL_HEADER, 0, includesMemstoreTS,
121         HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(), 0);
122     HFileBlock cacheBlock = blockEncoder.diskToCacheFormat(createBlockOnDisk(block), false);
123     assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length);
124   }
125 
126   private HFileBlock createBlockOnDisk(HFileBlock block) throws IOException {
127     int size;
128     HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
129         Compression.Algorithm.NONE, blockEncoder.getEncodingOnDisk(),
130         HConstants.HFILEBLOCK_DUMMY_HEADER);
131     context.setDummyHeader(block.getDummyHeaderForVersion());
132     blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(),
133             includesMemstoreTS, context, block.getBlockType());
134     byte[] encodedBytes = context.getUncompressedBytesWithHeader();
135     size = encodedBytes.length - block.getDummyHeaderForVersion().length;
136     return new HFileBlock(context.getBlockType(), size, size, -1,
137             ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, includesMemstoreTS,
138             block.getMinorVersion(), block.getBytesPerChecksum(), block.getChecksumType(),
139             block.getOnDiskDataSizeWithHeader());
140   }
141 
142   /**
143    * Test writing to disk.
144    * @throws IOException
145    */
146   @Test
147   public void testEncodingWritePath() throws IOException {
148     // usually we have just block without headers, but don't complicate that
149     HFileBlock block = getSampleHFileBlock();
150     HFileBlock blockOnDisk = createBlockOnDisk(block);
151 
152     if (blockEncoder.getEncodingOnDisk() !=
153         DataBlockEncoding.NONE) {
154       assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
155       assertEquals(blockEncoder.getEncodingOnDisk().getId(),
156           blockOnDisk.getDataBlockEncodingId());
157     } else {
158       assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
159     }
160   }
161 
162   /**
163    * Test converting blocks from disk to cache format.
164    */
165   @Test
166   public void testEncodingReadPath() {
167     HFileBlock origBlock = getSampleHFileBlock();
168     blockEncoder.diskToCacheFormat(origBlock, false);
169   }
170 
171   private HFileBlock getSampleHFileBlock() {
172     ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
173         generator.generateTestKeyValues(60), includesMemstoreTS);
174     int size = keyValues.limit();
175     ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE);
176     buf.position(HConstants.HFILEBLOCK_HEADER_SIZE);
177     keyValues.rewind();
178     buf.put(keyValues);
179     HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
180         HFileBlock.FILL_HEADER, 0, includesMemstoreTS, 
181         HFileReaderV2.MAX_MINOR_VERSION, 0, ChecksumType.NULL.getCode(), 0);
182     return b;
183   }
184 
185   /**
186    * @return All possible data block encoding configurations
187    */
188   @Parameters
189   public static Collection<Object[]> getAllConfigurations() {
190     List<Object[]> configurations =
191         new ArrayList<Object[]>();
192 
193     for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
194       for (DataBlockEncoding cacheAlgo : DataBlockEncoding.values()) {
195         if (diskAlgo != cacheAlgo && diskAlgo != DataBlockEncoding.NONE) {
196           // We allow (1) the same encoding on disk and in cache, and
197           // (2) some encoding in cache but no encoding on disk (for testing).
198           continue;
199         }
200         for (boolean includesMemstoreTS : new boolean[] {false, true}) {
201           configurations.add(new Object[] {
202               new HFileDataBlockEncoderImpl(diskAlgo, cacheAlgo),
203               new Boolean(includesMemstoreTS)});
204         }
205       }
206     }
207 
208     return configurations;
209   }
210 }