View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import static org.junit.Assert.*;
21  
22  import java.io.IOException;
23  import java.util.Random;
24  
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HBaseConfiguration;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.fs.HFileSystem;
34  import org.apache.hadoop.hbase.regionserver.StoreFile;
35  
36  import org.apache.hadoop.hbase.testclassification.SmallTests;
37  import org.junit.Before;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  
41  @Category(SmallTests.class)
42  public class TestPrefetch {
43  
44    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
45  
46    private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2;
47    private static final int DATA_BLOCK_SIZE = 2048;
48    private static final int NUM_KV = 1000;
49    private static final Random RNG = new Random();
50  
51    private Configuration conf;
52    private CacheConfig cacheConf;
53    private FileSystem fs;
54  
55    @Before
56    public void setUp() throws IOException {
57      conf = TEST_UTIL.getConfiguration();
58      conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
59      conf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true);
60      fs = HFileSystem.get(conf);
61      cacheConf = new CacheConfig(conf);
62    }
63  
64    @Test
65    public void testPrefetchSetInHCDWorks() {
66      HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("f"));
67      hcd.setPrefetchBlocksOnOpen(true);
68      Configuration c = HBaseConfiguration.create();
69      assertFalse(c.getBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, false));
70      CacheConfig cc = new CacheConfig(c, hcd);
71      assertTrue(cc.shouldPrefetchOnOpen());
72    }
73  
74    @Test(timeout=60000)
75    public void testPrefetch() throws Exception {
76      Path storeFile = writeStoreFile();
77      readStoreFile(storeFile);
78    }
79  
80    private void readStoreFile(Path storeFilePath) throws Exception {
81      // Open the file
82      HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs,
83        storeFilePath, cacheConf, conf);
84  
85      while (!((HFileReaderV3)reader).prefetchComplete()) {
86        // Sleep for a bit
87        Thread.sleep(1000);
88      }
89  
90      // Check that all of the data blocks were preloaded
91      BlockCache blockCache = cacheConf.getBlockCache();
92      long offset = 0;
93      HFileBlock prevBlock = null;
94      while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
95        long onDiskSize = -1;
96        if (prevBlock != null) {
97           onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
98        }
99        HFileBlock block = reader.readBlock(offset, onDiskSize, false, true, false, true, null);
100       BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), offset);
101       boolean isCached = blockCache.getBlock(blockCacheKey, true, false, true) != null;
102       if (block.getBlockType() == BlockType.DATA ||
103           block.getBlockType() == BlockType.ROOT_INDEX ||
104           block.getBlockType() == BlockType.INTERMEDIATE_INDEX) {
105         assertTrue(isCached);
106       }
107       prevBlock = block;
108       offset += block.getOnDiskSizeWithHeader();
109     }
110   }
111 
112   private Path writeStoreFile() throws IOException {
113     Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestPrefetch");
114     HFileContext meta = new HFileContextBuilder()
115       .withBlockSize(DATA_BLOCK_SIZE)
116       .build();
117     StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs)
118       .withOutputDir(storeFileParentDir)
119       .withComparator(KeyValue.COMPARATOR)
120       .withFileContext(meta)
121       .build();
122 
123     final int rowLen = 32;
124     for (int i = 0; i < NUM_KV; ++i) {
125       byte[] k = TestHFileWriterV2.randomOrderedKey(RNG, i);
126       byte[] v = TestHFileWriterV2.randomValue(RNG);
127       int cfLen = RNG.nextInt(k.length - rowLen + 1);
128       KeyValue kv = new KeyValue(
129           k, 0, rowLen,
130           k, rowLen, cfLen,
131           k, rowLen + cfLen, k.length - rowLen - cfLen,
132           RNG.nextLong(),
133           generateKeyType(RNG),
134           v, 0, v.length);
135       sfw.append(kv);
136     }
137 
138     sfw.close();
139     return sfw.getPath();
140   }
141 
142   public static KeyValue.Type generateKeyType(Random rand) {
143     if (rand.nextBoolean()) {
144       // Let's make half of KVs puts.
145       return KeyValue.Type.Put;
146     } else {
147       KeyValue.Type keyType =
148           KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
149       if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum)
150       {
151         throw new RuntimeException("Generated an invalid key type: " + keyType
152             + ". " + "Probably the layout of KeyValue.Type has changed.");
153       }
154       return keyType;
155     }
156   }
157 
158 }