View Javadoc

1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import static org.junit.Assert.*;
23  
24  import java.io.IOException;
25  import java.util.Random;
26  
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.KeyValue;
32  import org.apache.hadoop.hbase.testclassification.SmallTests;
33  import org.apache.hadoop.hbase.fs.HFileSystem;
34  import org.apache.hadoop.hbase.regionserver.StoreFile;
35  
36  import org.junit.Before;
37  import org.junit.Test;
38  import org.junit.experimental.categories.Category;
39  
40  @Category(SmallTests.class)
41  public class TestPrefetch {
42  
43    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
44  
45    private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2;
46    private static final int DATA_BLOCK_SIZE = 2048;
47    private static final int NUM_KV = 1000;
48    private static final Random RNG = new Random();
49  
50    private Configuration conf;
51    private CacheConfig cacheConf;
52    private FileSystem fs;
53  
54    @Before
55    public void setUp() throws IOException {
56      conf = TEST_UTIL.getConfiguration();
57      conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
58      conf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true);
59      fs = HFileSystem.get(conf);
60      CacheConfig.blockCacheDisabled = false;
61      cacheConf = new CacheConfig(conf);
62    }
63  
64    @Test(timeout=60000)
65    public void testPrefetch() throws Exception {
66      Path storeFile = writeStoreFile();
67      readStoreFile(storeFile);
68    }
69  
70    private void readStoreFile(Path storeFilePath) throws Exception {
71      // Open the file
72      HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs,
73        storeFilePath, cacheConf, conf);
74  
75      while (!((HFileReaderV3)reader).prefetchComplete()) {
76        // Sleep for a bit
77        Thread.sleep(1000);
78      }
79  
80      // Check that all of the data blocks were preloaded
81      BlockCache blockCache = cacheConf.getBlockCache();
82      long offset = 0;
83      HFileBlock prevBlock = null;
84      while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
85        long onDiskSize = -1;
86        if (prevBlock != null) {
87           onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
88        }
89        HFileBlock block = reader.readBlock(offset, onDiskSize, false, true, false, true, null,
90          null);
91        BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), offset);
92        boolean isCached = blockCache.getBlock(blockCacheKey, true, false, true) != null;
93        if (block.getBlockType() == BlockType.DATA ||
94            block.getBlockType() == BlockType.ROOT_INDEX ||
95            block.getBlockType() == BlockType.INTERMEDIATE_INDEX) {
96          assertTrue(isCached);
97        }
98        prevBlock = block;
99        offset += block.getOnDiskSizeWithHeader();
100     }
101   }
102 
103   private Path writeStoreFile() throws IOException {
104     Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestPrefetch");
105     HFileContext meta = new HFileContextBuilder()
106       .withBlockSize(DATA_BLOCK_SIZE)
107       .build();
108     StoreFile.Writer sfw = new StoreFile.WriterBuilder(conf, cacheConf, fs)
109       .withOutputDir(storeFileParentDir)
110       .withComparator(KeyValue.COMPARATOR)
111       .withFileContext(meta)
112       .build();
113 
114     final int rowLen = 32;
115     for (int i = 0; i < NUM_KV; ++i) {
116       byte[] k = TestHFileWriterV2.randomOrderedKey(RNG, i);
117       byte[] v = TestHFileWriterV2.randomValue(RNG);
118       int cfLen = RNG.nextInt(k.length - rowLen + 1);
119       KeyValue kv = new KeyValue(
120           k, 0, rowLen,
121           k, rowLen, cfLen,
122           k, rowLen + cfLen, k.length - rowLen - cfLen,
123           RNG.nextLong(),
124           generateKeyType(RNG),
125           v, 0, v.length);
126       sfw.append(kv);
127     }
128 
129     sfw.close();
130     return sfw.getPath();
131   }
132 
133   public static KeyValue.Type generateKeyType(Random rand) {
134     if (rand.nextBoolean()) {
135       // Let's make half of KVs puts.
136       return KeyValue.Type.Put;
137     } else {
138       KeyValue.Type keyType =
139           KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
140       if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum)
141       {
142         throw new RuntimeException("Generated an invalid key type: " + keyType
143             + ". " + "Probably the layout of KeyValue.Type has changed.");
144       }
145       return keyType;
146     }
147   }
148 
149 }