1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.hfile;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertTrue;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.List;
27
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.SmallTests;
32 import org.apache.hadoop.hbase.io.HeapSize;
33 import org.apache.hadoop.hbase.io.compress.Compression;
34 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
35 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
36 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
37 import org.apache.hadoop.hbase.util.ChecksumType;
38 import org.apache.hadoop.hbase.util.test.RedundantKVGenerator;
39 import org.junit.Before;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42 import org.junit.runner.RunWith;
43 import org.junit.runners.Parameterized;
44 import org.junit.runners.Parameterized.Parameters;
45
46 @RunWith(Parameterized.class)
47 @Category(SmallTests.class)
48 public class TestHFileDataBlockEncoder {
49 private Configuration conf;
50 private final HBaseTestingUtility TEST_UTIL =
51 new HBaseTestingUtility();
52 private HFileDataBlockEncoderImpl blockEncoder;
53 private RedundantKVGenerator generator = new RedundantKVGenerator();
54 private boolean includesMemstoreTS;
55
56
57
58
59
60 public TestHFileDataBlockEncoder(HFileDataBlockEncoderImpl blockEncoder,
61 boolean includesMemstoreTS) {
62 this.blockEncoder = blockEncoder;
63 this.includesMemstoreTS = includesMemstoreTS;
64 System.err.println("On-disk encoding: " + blockEncoder.getEncodingOnDisk()
65 + ", in-cache encoding: " + blockEncoder.getEncodingInCache()
66 + ", includesMemstoreTS: " + includesMemstoreTS);
67 }
68
69
70
71
72 @Before
73 public void setUp() {
74 conf = TEST_UTIL.getConfiguration();
75 }
76
77
78
79
80
81 @Test
82 public void testEncodingWithCache() {
83 HFileBlock block = getSampleHFileBlock();
84 LruBlockCache blockCache =
85 new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
86 HFileBlock cacheBlock = blockEncoder.diskToCacheFormat(block, false);
87 BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
88 blockCache.cacheBlock(cacheKey, cacheBlock);
89
90 HeapSize heapSize = blockCache.getBlock(cacheKey, false, false);
91 assertTrue(heapSize instanceof HFileBlock);
92
93 HFileBlock returnedBlock = (HFileBlock) heapSize;;
94
95 if (blockEncoder.getEncodingInCache() ==
96 DataBlockEncoding.NONE) {
97 assertEquals(block.getBufferWithHeader(),
98 returnedBlock.getBufferWithHeader());
99 } else {
100 if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
101 System.out.println(blockEncoder);
102 }
103 assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
104 }
105 }
106
107
108 @Test
109 public void testHeaderSizeInCacheWithoutChecksum() throws Exception {
110 int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
111
112 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
113 generator.generateTestKeyValues(60), includesMemstoreTS);
114 int size = keyValues.limit();
115 ByteBuffer buf = ByteBuffer.allocate(size + headerSize);
116 buf.position(headerSize);
117 keyValues.rewind();
118 buf.put(keyValues);
119 HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
120 HFileBlock.FILL_HEADER, 0, includesMemstoreTS,
121 HFileBlock.MINOR_VERSION_NO_CHECKSUM, 0, ChecksumType.NULL.getCode(), 0);
122 HFileBlock cacheBlock = blockEncoder.diskToCacheFormat(createBlockOnDisk(block), false);
123 assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length);
124 }
125
126 private HFileBlock createBlockOnDisk(HFileBlock block) throws IOException {
127 int size;
128 HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
129 Compression.Algorithm.NONE, blockEncoder.getEncodingOnDisk(),
130 HConstants.HFILEBLOCK_DUMMY_HEADER);
131 context.setDummyHeader(block.getDummyHeaderForVersion());
132 blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(),
133 includesMemstoreTS, context, block.getBlockType());
134 byte[] encodedBytes = context.getUncompressedBytesWithHeader();
135 size = encodedBytes.length - block.getDummyHeaderForVersion().length;
136 return new HFileBlock(context.getBlockType(), size, size, -1,
137 ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, includesMemstoreTS,
138 block.getMinorVersion(), block.getBytesPerChecksum(), block.getChecksumType(),
139 block.getOnDiskDataSizeWithHeader());
140 }
141
142
143
144
145
146 @Test
147 public void testEncodingWritePath() throws IOException {
148
149 HFileBlock block = getSampleHFileBlock();
150 HFileBlock blockOnDisk = createBlockOnDisk(block);
151
152 if (blockEncoder.getEncodingOnDisk() !=
153 DataBlockEncoding.NONE) {
154 assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
155 assertEquals(blockEncoder.getEncodingOnDisk().getId(),
156 blockOnDisk.getDataBlockEncodingId());
157 } else {
158 assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
159 }
160 }
161
162
163
164
165 @Test
166 public void testEncodingReadPath() {
167 HFileBlock origBlock = getSampleHFileBlock();
168 blockEncoder.diskToCacheFormat(origBlock, false);
169 }
170
171 private HFileBlock getSampleHFileBlock() {
172 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
173 generator.generateTestKeyValues(60), includesMemstoreTS);
174 int size = keyValues.limit();
175 ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE);
176 buf.position(HConstants.HFILEBLOCK_HEADER_SIZE);
177 keyValues.rewind();
178 buf.put(keyValues);
179 HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
180 HFileBlock.FILL_HEADER, 0, includesMemstoreTS,
181 HFileReaderV2.MAX_MINOR_VERSION, 0, ChecksumType.NULL.getCode(), 0);
182 return b;
183 }
184
185
186
187
188 @Parameters
189 public static Collection<Object[]> getAllConfigurations() {
190 List<Object[]> configurations =
191 new ArrayList<Object[]>();
192
193 for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
194 for (DataBlockEncoding cacheAlgo : DataBlockEncoding.values()) {
195 if (diskAlgo != cacheAlgo && diskAlgo != DataBlockEncoding.NONE) {
196
197
198 continue;
199 }
200 for (boolean includesMemstoreTS : new boolean[] {false, true}) {
201 configurations.add(new Object[] {
202 new HFileDataBlockEncoderImpl(diskAlgo, cacheAlgo),
203 new Boolean(includesMemstoreTS)});
204 }
205 }
206 }
207
208 return configurations;
209 }
210 }