1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertFalse;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.Collection;
30 import java.util.EnumMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Random;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.Cell;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.HColumnDescriptor;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.KeyValue;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.Tag;
47 import org.apache.hadoop.hbase.client.Durability;
48 import org.apache.hadoop.hbase.client.Put;
49 import org.apache.hadoop.hbase.fs.HFileSystem;
50 import org.apache.hadoop.hbase.io.compress.Compression;
51 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
52 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
53 import org.apache.hadoop.hbase.regionserver.BloomType;
54 import org.apache.hadoop.hbase.regionserver.HRegion;
55 import org.apache.hadoop.hbase.regionserver.StoreFile;
56 import org.apache.hadoop.hbase.util.BloomFilterFactory;
57 import org.apache.hadoop.hbase.util.Bytes;
58 import org.apache.hadoop.hbase.util.ChecksumType;
59 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
60 import org.junit.After;
61 import org.junit.Before;
62 import org.junit.Test;
63 import org.junit.experimental.categories.Category;
64 import org.junit.runner.RunWith;
65 import org.junit.runners.Parameterized;
66 import org.junit.runners.Parameterized.Parameters;
67
68
69
70
71
72 @RunWith(Parameterized.class)
73 @Category(MediumTests.class)
74 public class TestCacheOnWrite {
75
76 private static final Log LOG = LogFactory.getLog(TestCacheOnWrite.class);
77
78 private static final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
79 private Configuration conf;
80 private CacheConfig cacheConf;
81 private FileSystem fs;
82 private Random rand = new Random(12983177L);
83 private Path storeFilePath;
84 private BlockCache blockCache;
85 private String testDescription;
86
87 private final CacheOnWriteType cowType;
88 private final Compression.Algorithm compress;
89 private final BlockEncoderTestType encoderType;
90 private final HFileDataBlockEncoder encoder;
91 private final boolean cacheCompressedData;
92
93 private static final int DATA_BLOCK_SIZE = 2048;
94 private static final int NUM_KV = 25000;
95 private static final int INDEX_BLOCK_SIZE = 512;
96 private static final int BLOOM_BLOCK_SIZE = 4096;
97 private static final BloomType BLOOM_TYPE = BloomType.ROWCOL;
98 private static final int CKBYTES = 512;
99
100
101 private static final int NUM_VALID_KEY_TYPES =
102 KeyValue.Type.values().length - 2;
103
104 private static enum CacheOnWriteType {
105 DATA_BLOCKS(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY,
106 BlockType.DATA, BlockType.ENCODED_DATA),
107 BLOOM_BLOCKS(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
108 BlockType.BLOOM_CHUNK),
109 INDEX_BLOCKS(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
110 BlockType.LEAF_INDEX, BlockType.INTERMEDIATE_INDEX);
111
112 private final String confKey;
113 private final BlockType blockType1;
114 private final BlockType blockType2;
115
116 private CacheOnWriteType(String confKey, BlockType blockType) {
117 this(confKey, blockType, blockType);
118 }
119
120 private CacheOnWriteType(String confKey, BlockType blockType1,
121 BlockType blockType2) {
122 this.blockType1 = blockType1;
123 this.blockType2 = blockType2;
124 this.confKey = confKey;
125 }
126
127 public boolean shouldBeCached(BlockType blockType) {
128 return blockType == blockType1 || blockType == blockType2;
129 }
130
131 public void modifyConf(Configuration conf) {
132 for (CacheOnWriteType cowType : CacheOnWriteType.values()) {
133 conf.setBoolean(cowType.confKey, cowType == this);
134 }
135 }
136
137 }
138
139 private static final DataBlockEncoding ENCODING_ALGO =
140 DataBlockEncoding.PREFIX;
141
142
143 private static enum BlockEncoderTestType {
144 NO_BLOCK_ENCODING_NOOP(true, false),
145 NO_BLOCK_ENCODING(false, false),
146 BLOCK_ENCODING_EVERYWHERE(false, true);
147
148 private final boolean noop;
149 private final boolean encode;
150
151 BlockEncoderTestType(boolean noop, boolean encode) {
152 this.encode = encode;
153 this.noop = noop;
154 }
155
156 public HFileDataBlockEncoder getEncoder() {
157 return noop ? NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(
158 encode ? ENCODING_ALGO : DataBlockEncoding.NONE);
159 }
160 }
161
162 public TestCacheOnWrite(CacheOnWriteType cowType, Compression.Algorithm compress,
163 BlockEncoderTestType encoderType, boolean cacheCompressedData, BlockCache blockCache) {
164 this.cowType = cowType;
165 this.compress = compress;
166 this.encoderType = encoderType;
167 this.encoder = encoderType.getEncoder();
168 this.cacheCompressedData = cacheCompressedData;
169 this.blockCache = blockCache;
170 testDescription = "[cacheOnWrite=" + cowType + ", compress=" + compress +
171 ", encoderType=" + encoderType + ", cacheCompressedData=" + cacheCompressedData +
172 ", blockCache=" + blockCache.getClass().getSimpleName() + "]";
173 System.out.println(testDescription);
174 }
175
176 private static List<BlockCache> getBlockCaches() throws IOException {
177 Configuration conf = TEST_UTIL.getConfiguration();
178 List<BlockCache> blockcaches = new ArrayList<BlockCache>();
179
180 blockcaches.add(new CacheConfig(conf).getBlockCache());
181
182
183 BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024, TEST_UTIL.getConfiguration());
184 blockcaches.add(lru);
185
186
187 FileSystem.get(conf).mkdirs(TEST_UTIL.getDataTestDir());
188 int[] bucketSizes = {INDEX_BLOCK_SIZE, DATA_BLOCK_SIZE, BLOOM_BLOCK_SIZE, 64 * 1024 };
189 BlockCache bucketcache =
190 new BucketCache("file:" + TEST_UTIL.getDataTestDir() + "/bucket.data",
191 256 * 1024 * 1024, 64 * 1024, bucketSizes, 3, 256 * 1024, null);
192 blockcaches.add(bucketcache);
193 return blockcaches;
194 }
195
196 @Parameters
197 public static Collection<Object[]> getParameters() throws IOException {
198 List<Object[]> cowTypes = new ArrayList<Object[]>();
199 for (BlockCache blockache : getBlockCaches()) {
200 for (CacheOnWriteType cowType : CacheOnWriteType.values()) {
201 for (Compression.Algorithm compress : HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
202 for (BlockEncoderTestType encoderType : BlockEncoderTestType.values()) {
203 for (boolean cacheCompressedData : new boolean[] { false, true }) {
204 cowTypes.add(new Object[] { cowType, compress, encoderType, cacheCompressedData, blockache});
205 }
206 }
207 }
208 }
209 }
210 return cowTypes;
211 }
212
213 @Before
214 public void setUp() throws IOException {
215 conf = TEST_UTIL.getConfiguration();
216 this.conf.set("dfs.datanode.data.dir.perm", "700");
217 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
218 conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, INDEX_BLOCK_SIZE);
219 conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE,
220 BLOOM_BLOCK_SIZE);
221 conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, cacheCompressedData);
222 cowType.modifyConf(conf);
223 fs = HFileSystem.get(conf);
224 cacheConf =
225 new CacheConfig(blockCache, true, true, cowType.shouldBeCached(BlockType.DATA),
226 cowType.shouldBeCached(BlockType.LEAF_INDEX),
227 cowType.shouldBeCached(BlockType.BLOOM_CHUNK), false, cacheCompressedData, true);
228 }
229
230 @After
231 public void tearDown() {
232 }
233
234 @Test
235 public void testStoreFileCacheOnWrite() throws IOException {
236 testStoreFileCacheOnWriteInternals(false);
237 testStoreFileCacheOnWriteInternals(true);
238 }
239
240 protected void testStoreFileCacheOnWriteInternals(boolean useTags) throws IOException {
241 writeStoreFile(useTags);
242 readStoreFile(useTags);
243 }
244
245 private void readStoreFile(boolean useTags) throws IOException {
246 AbstractHFileReader reader;
247 if (useTags) {
248 reader = (HFileReaderV3) HFile.createReader(fs, storeFilePath, cacheConf, conf);
249 } else {
250 reader = (HFileReaderV2) HFile.createReader(fs, storeFilePath, cacheConf, conf);
251 }
252 LOG.info("HFile information: " + reader);
253 HFileContext meta = new HFileContextBuilder().withCompression(compress)
254 .withBytesPerCheckSum(CKBYTES).withChecksumType(ChecksumType.NULL)
255 .withBlockSize(DATA_BLOCK_SIZE).withDataBlockEncoding(encoder.getDataBlockEncoding())
256 .withIncludesTags(useTags).build();
257 final boolean cacheBlocks = false;
258 final boolean pread = false;
259 HFileScanner scanner = reader.getScanner(cacheBlocks, pread);
260 assertTrue(testDescription, scanner.seekTo());
261
262 long offset = 0;
263 HFileBlock prevBlock = null;
264 EnumMap<BlockType, Integer> blockCountByType =
265 new EnumMap<BlockType, Integer>(BlockType.class);
266
267 DataBlockEncoding encodingInCache =
268 encoderType.getEncoder().getDataBlockEncoding();
269 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
270 long onDiskSize = -1;
271 if (prevBlock != null) {
272 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
273 }
274
275
276 HFileBlock block = reader.readBlock(offset, onDiskSize, false, true,
277 false, true, null);
278 BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
279 offset, encodingInCache, block.getBlockType());
280 HFileBlock fromCache = (HFileBlock) blockCache.getBlock(blockCacheKey, true, false, true);
281 boolean isCached = fromCache != null;
282 boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
283 assertTrue("shouldBeCached: " + shouldBeCached+ "\n" +
284 "isCached: " + isCached + "\n" +
285 "Test description: " + testDescription + "\n" +
286 "block: " + block + "\n" +
287 "encodingInCache: " + encodingInCache + "\n" +
288 "blockCacheKey: " + blockCacheKey,
289 shouldBeCached == isCached);
290 if (isCached) {
291 if (cacheConf.shouldCacheCompressed(fromCache.getBlockType().getCategory())) {
292 if (compress != Compression.Algorithm.NONE) {
293 assertFalse(fromCache.isUnpacked());
294 }
295 fromCache = fromCache.unpack(meta, reader.getUncachedBlockReader());
296 } else {
297 assertTrue(fromCache.isUnpacked());
298 }
299
300 assertEquals(block.getChecksumType(), fromCache.getChecksumType());
301 assertEquals(block.getBlockType(), fromCache.getBlockType());
302 if (block.getBlockType() == BlockType.ENCODED_DATA) {
303 assertEquals(block.getDataBlockEncodingId(), fromCache.getDataBlockEncodingId());
304 assertEquals(block.getDataBlockEncoding(), fromCache.getDataBlockEncoding());
305 }
306 assertEquals(block.getOnDiskSizeWithHeader(), fromCache.getOnDiskSizeWithHeader());
307 assertEquals(block.getOnDiskSizeWithoutHeader(), fromCache.getOnDiskSizeWithoutHeader());
308 assertEquals(
309 block.getUncompressedSizeWithoutHeader(), fromCache.getUncompressedSizeWithoutHeader());
310 }
311 prevBlock = block;
312 offset += block.getOnDiskSizeWithHeader();
313 BlockType bt = block.getBlockType();
314 Integer count = blockCountByType.get(bt);
315 blockCountByType.put(bt, (count == null ? 0 : count) + 1);
316 }
317
318 LOG.info("Block count by type: " + blockCountByType);
319 String countByType = blockCountByType.toString();
320 BlockType cachedDataBlockType =
321 encoderType.encode ? BlockType.ENCODED_DATA : BlockType.DATA;
322 if (useTags) {
323 assertEquals("{" + cachedDataBlockType
324 + "=1550, LEAF_INDEX=173, BLOOM_CHUNK=9, INTERMEDIATE_INDEX=20}", countByType);
325 } else {
326 assertEquals("{" + cachedDataBlockType
327 + "=1379, LEAF_INDEX=154, BLOOM_CHUNK=9, INTERMEDIATE_INDEX=18}", countByType);
328 }
329
330
331 while (scanner.next()) {
332 Cell cell = scanner.getKeyValue();
333 }
334 reader.close();
335 }
336
337 public static KeyValue.Type generateKeyType(Random rand) {
338 if (rand.nextBoolean()) {
339
340 return KeyValue.Type.Put;
341 } else {
342 KeyValue.Type keyType =
343 KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
344 if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum)
345 {
346 throw new RuntimeException("Generated an invalid key type: " + keyType
347 + ". " + "Probably the layout of KeyValue.Type has changed.");
348 }
349 return keyType;
350 }
351 }
352
353 public void writeStoreFile(boolean useTags) throws IOException {
354 if(useTags) {
355 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
356 } else {
357 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 2);
358 }
359 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(),
360 "test_cache_on_write");
361 HFileContext meta = new HFileContextBuilder().withCompression(compress)
362 .withBytesPerCheckSum(CKBYTES).withChecksumType(ChecksumType.NULL)
363 .withBlockSize(DATA_BLOCK_SIZE).withDataBlockEncoding(encoder.getDataBlockEncoding())
364 .withIncludesTags(useTags).build();
365 StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs)
366 .withOutputDir(storeFileParentDir).withComparator(KeyValue.COMPARATOR)
367 .withFileContext(meta)
368 .withBloomType(BLOOM_TYPE).withMaxKeyCount(NUM_KV).build();
369
370 final int rowLen = 32;
371 for (int i = 0; i < NUM_KV; ++i) {
372 byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i);
373 byte[] v = TestHFileWriterV2.randomValue(rand);
374 int cfLen = rand.nextInt(k.length - rowLen + 1);
375 KeyValue kv;
376 if(useTags) {
377 Tag t = new Tag((byte) 1, "visibility");
378 List<Tag> tagList = new ArrayList<Tag>();
379 tagList.add(t);
380 Tag[] tags = new Tag[1];
381 tags[0] = t;
382 kv = new KeyValue(
383 k, 0, rowLen,
384 k, rowLen, cfLen,
385 k, rowLen + cfLen, k.length - rowLen - cfLen,
386 rand.nextLong(),
387 generateKeyType(rand),
388 v, 0, v.length, tagList);
389 } else {
390 kv = new KeyValue(
391 k, 0, rowLen,
392 k, rowLen, cfLen,
393 k, rowLen + cfLen, k.length - rowLen - cfLen,
394 rand.nextLong(),
395 generateKeyType(rand),
396 v, 0, v.length);
397 }
398 sfw.append(kv);
399 }
400
401 sfw.close();
402 storeFilePath = sfw.getPath();
403 }
404
405 @Test
406 public void testNotCachingDataBlocksDuringCompaction() throws IOException {
407 testNotCachingDataBlocksDuringCompactionInternals(false);
408 testNotCachingDataBlocksDuringCompactionInternals(true);
409 }
410
411 protected void testNotCachingDataBlocksDuringCompactionInternals(boolean useTags) throws IOException {
412 if (useTags) {
413 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
414 } else {
415 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 2);
416 }
417
418
419
420 final String table = "CompactionCacheOnWrite";
421 final String cf = "myCF";
422 final byte[] cfBytes = Bytes.toBytes(cf);
423 final int maxVersions = 3;
424 HRegion region = TEST_UTIL.createTestRegion(table,
425 new HColumnDescriptor(cf)
426 .setCompressionType(compress)
427 .setBloomFilterType(BLOOM_TYPE)
428 .setMaxVersions(maxVersions)
429 .setDataBlockEncoding(encoder.getDataBlockEncoding())
430 );
431 int rowIdx = 0;
432 long ts = EnvironmentEdgeManager.currentTimeMillis();
433 for (int iFile = 0; iFile < 5; ++iFile) {
434 for (int iRow = 0; iRow < 500; ++iRow) {
435 String rowStr = "" + (rowIdx * rowIdx * rowIdx) + "row" + iFile + "_" +
436 iRow;
437 Put p = new Put(Bytes.toBytes(rowStr));
438 ++rowIdx;
439 for (int iCol = 0; iCol < 10; ++iCol) {
440 String qualStr = "col" + iCol;
441 String valueStr = "value_" + rowStr + "_" + qualStr;
442 for (int iTS = 0; iTS < 5; ++iTS) {
443 if (useTags) {
444 Tag t = new Tag((byte) 1, "visibility");
445 Tag[] tags = new Tag[1];
446 tags[0] = t;
447 KeyValue kv = new KeyValue(Bytes.toBytes(rowStr), cfBytes, Bytes.toBytes(qualStr),
448 HConstants.LATEST_TIMESTAMP, Bytes.toBytes(valueStr), tags);
449 p.add(kv);
450 } else {
451 p.add(cfBytes, Bytes.toBytes(qualStr), ts++, Bytes.toBytes(valueStr));
452 }
453 }
454 }
455 p.setDurability(Durability.ASYNC_WAL);
456 region.put(p);
457 }
458 region.flushcache();
459 }
460 LruBlockCache blockCache =
461 (LruBlockCache) new CacheConfig(conf).getBlockCache();
462 blockCache.clearCache();
463 assertEquals(0, blockCache.getBlockTypeCountsForTest().size());
464 region.compactStores();
465 LOG.debug("compactStores() returned");
466
467 Map<BlockType, Integer> blockTypesInCache =
468 blockCache.getBlockTypeCountsForTest();
469 LOG.debug("Block types in cache: " + blockTypesInCache);
470 assertNull(blockTypesInCache.get(BlockType.ENCODED_DATA));
471 assertNull(blockTypesInCache.get(BlockType.DATA));
472 region.close();
473 blockCache.shutdown();
474 }
475 }
476