1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.junit.Assert.*;
23
24 import java.io.IOException;
25 import java.util.Random;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.KeyValue;
32 import org.apache.hadoop.hbase.testclassification.SmallTests;
33 import org.apache.hadoop.hbase.fs.HFileSystem;
34 import org.apache.hadoop.hbase.regionserver.StoreFile;
35
36 import org.junit.Before;
37 import org.junit.Test;
38 import org.junit.experimental.categories.Category;
39
40 @Category(SmallTests.class)
41 public class TestPrefetch {
42
43 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
44
45 private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2;
46 private static final int DATA_BLOCK_SIZE = 2048;
47 private static final int NUM_KV = 1000;
48 private static final Random RNG = new Random();
49
50 private Configuration conf;
51 private CacheConfig cacheConf;
52 private FileSystem fs;
53
54 @Before
55 public void setUp() throws IOException {
56 conf = TEST_UTIL.getConfiguration();
57 conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
58 conf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true);
59 fs = HFileSystem.get(conf);
60 CacheConfig.blockCacheDisabled = false;
61 cacheConf = new CacheConfig(conf);
62 }
63
64 @Test(timeout=60000)
65 public void testPrefetch() throws Exception {
66 Path storeFile = writeStoreFile();
67 readStoreFile(storeFile);
68 }
69
70 private void readStoreFile(Path storeFilePath) throws Exception {
71
72 HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs,
73 storeFilePath, cacheConf, conf);
74
75 while (!((HFileReaderV3)reader).prefetchComplete()) {
76
77 Thread.sleep(1000);
78 }
79
80
81 BlockCache blockCache = cacheConf.getBlockCache();
82 long offset = 0;
83 HFileBlock prevBlock = null;
84 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
85 long onDiskSize = -1;
86 if (prevBlock != null) {
87 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
88 }
89 HFileBlock block = reader.readBlock(offset, onDiskSize, false, true, false, true, null,
90 null);
91 BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), offset);
92 boolean isCached = blockCache.getBlock(blockCacheKey, true, false, true) != null;
93 if (block.getBlockType() == BlockType.DATA ||
94 block.getBlockType() == BlockType.ROOT_INDEX ||
95 block.getBlockType() == BlockType.INTERMEDIATE_INDEX) {
96 assertTrue(isCached);
97 }
98 prevBlock = block;
99 offset += block.getOnDiskSizeWithHeader();
100 }
101 }
102
103 private Path writeStoreFile() throws IOException {
104 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestPrefetch");
105 HFileContext meta = new HFileContextBuilder()
106 .withBlockSize(DATA_BLOCK_SIZE)
107 .build();
108 StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs)
109 .withOutputDir(storeFileParentDir)
110 .withComparator(KeyValue.COMPARATOR)
111 .withFileContext(meta)
112 .build();
113
114 final int rowLen = 32;
115 for (int i = 0; i < NUM_KV; ++i) {
116 byte[] k = TestHFileWriterV2.randomOrderedKey(RNG, i);
117 byte[] v = TestHFileWriterV2.randomValue(RNG);
118 int cfLen = RNG.nextInt(k.length - rowLen + 1);
119 KeyValue kv = new KeyValue(
120 k, 0, rowLen,
121 k, rowLen, cfLen,
122 k, rowLen + cfLen, k.length - rowLen - cfLen,
123 RNG.nextLong(),
124 generateKeyType(RNG),
125 v, 0, v.length);
126 sfw.append(kv);
127 }
128
129 sfw.close();
130 return sfw.getPath();
131 }
132
133 public static KeyValue.Type generateKeyType(Random rand) {
134 if (rand.nextBoolean()) {
135
136 return KeyValue.Type.Put;
137 } else {
138 KeyValue.Type keyType =
139 KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
140 if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum)
141 {
142 throw new RuntimeException("Generated an invalid key type: " + keyType
143 + ". " + "Probably the layout of KeyValue.Type has changed.");
144 }
145 return keyType;
146 }
147 }
148
149 }