1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import static org.junit.Assert.*;
21
22 import java.io.IOException;
23 import java.util.Random;
24
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HBaseConfiguration;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.util.Bytes;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.fs.HFileSystem;
34 import org.apache.hadoop.hbase.regionserver.StoreFile;
35
36 import org.apache.hadoop.hbase.testclassification.SmallTests;
37 import org.junit.Before;
38 import org.junit.Test;
39 import org.junit.experimental.categories.Category;
40
41 @Category(SmallTests.class)
42 public class TestPrefetch {
43
44 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
45
46 private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2;
47 private static final int DATA_BLOCK_SIZE = 2048;
48 private static final int NUM_KV = 1000;
49 private static final Random RNG = new Random();
50
51 private Configuration conf;
52 private CacheConfig cacheConf;
53 private FileSystem fs;
54
55 @Before
56 public void setUp() throws IOException {
57 conf = TEST_UTIL.getConfiguration();
58 conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
59 conf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true);
60 fs = HFileSystem.get(conf);
61 cacheConf = new CacheConfig(conf);
62 }
63
64 @Test
65 public void testPrefetchSetInHCDWorks() {
66 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("f"));
67 hcd.setPrefetchBlocksOnOpen(true);
68 Configuration c = HBaseConfiguration.create();
69 assertFalse(c.getBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, false));
70 CacheConfig cc = new CacheConfig(c, hcd);
71 assertTrue(cc.shouldPrefetchOnOpen());
72 }
73
74 @Test(timeout=60000)
75 public void testPrefetch() throws Exception {
76 Path storeFile = writeStoreFile();
77 readStoreFile(storeFile);
78 }
79
80 private void readStoreFile(Path storeFilePath) throws Exception {
81
82 HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs,
83 storeFilePath, cacheConf, conf);
84
85 while (!((HFileReaderV3)reader).prefetchComplete()) {
86
87 Thread.sleep(1000);
88 }
89
90
91 BlockCache blockCache = cacheConf.getBlockCache();
92 long offset = 0;
93 HFileBlock prevBlock = null;
94 while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
95 long onDiskSize = -1;
96 if (prevBlock != null) {
97 onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
98 }
99 HFileBlock block = reader.readBlock(offset, onDiskSize, false, true, false, true, null);
100 BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), offset);
101 boolean isCached = blockCache.getBlock(blockCacheKey, true, false, true) != null;
102 if (block.getBlockType() == BlockType.DATA ||
103 block.getBlockType() == BlockType.ROOT_INDEX ||
104 block.getBlockType() == BlockType.INTERMEDIATE_INDEX) {
105 assertTrue(isCached);
106 }
107 prevBlock = block;
108 offset += block.getOnDiskSizeWithHeader();
109 }
110 }
111
112 private Path writeStoreFile() throws IOException {
113 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestPrefetch");
114 HFileContext meta = new HFileContextBuilder()
115 .withBlockSize(DATA_BLOCK_SIZE)
116 .build();
117 StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs)
118 .withOutputDir(storeFileParentDir)
119 .withComparator(KeyValue.COMPARATOR)
120 .withFileContext(meta)
121 .build();
122
123 final int rowLen = 32;
124 for (int i = 0; i < NUM_KV; ++i) {
125 byte[] k = TestHFileWriterV2.randomOrderedKey(RNG, i);
126 byte[] v = TestHFileWriterV2.randomValue(RNG);
127 int cfLen = RNG.nextInt(k.length - rowLen + 1);
128 KeyValue kv = new KeyValue(
129 k, 0, rowLen,
130 k, rowLen, cfLen,
131 k, rowLen + cfLen, k.length - rowLen - cfLen,
132 RNG.nextLong(),
133 generateKeyType(RNG),
134 v, 0, v.length);
135 sfw.append(kv);
136 }
137
138 sfw.close();
139 return sfw.getPath();
140 }
141
142 public static KeyValue.Type generateKeyType(Random rand) {
143 if (rand.nextBoolean()) {
144
145 return KeyValue.Type.Put;
146 } else {
147 KeyValue.Type keyType =
148 KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
149 if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum)
150 {
151 throw new RuntimeException("Generated an invalid key type: " + keyType
152 + ". " + "Probably the layout of KeyValue.Type has changed.");
153 }
154 return keyType;
155 }
156 }
157
158 }