View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.regionserver;
18  
19  import java.io.IOException;
20  import java.util.ArrayList;
21  import java.util.List;
22  import java.util.Random;
23  
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.Path;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.KeyValue;
29  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
30  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
31  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
32  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
33  import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
34  import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
35  
36  /**
37   * Test seek performance for encoded data blocks. Read an HFile and do several
38   * random seeks.
39   */
40  public class EncodedSeekPerformanceTest {
41    private static final double NANOSEC_IN_SEC = 1000.0 * 1000.0 * 1000.0;
42    private static final double BYTES_IN_MEGABYTES = 1024.0 * 1024.0;
43    /** Default number of seeks which will be used in benchmark. */
44    public static int DEFAULT_NUMBER_OF_SEEKS = 10000;
45  
46    private final HBaseTestingUtility testingUtility = new HBaseTestingUtility();
47    private Configuration configuration = testingUtility.getConfiguration();
48    private CacheConfig cacheConf = new CacheConfig(configuration);
49    private Random randomizer;
50    private int numberOfSeeks;
51  
52    /** Use this benchmark with default options */
53    public EncodedSeekPerformanceTest() {
54      configuration.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f);
55      randomizer = new Random(42l);
56      numberOfSeeks = DEFAULT_NUMBER_OF_SEEKS;
57    }
58  
59    private List<KeyValue> prepareListOfTestSeeks(Path path) throws IOException {
60      List<KeyValue> allKeyValues = new ArrayList<KeyValue>();
61  
62      // read all of the key values
63      StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
64          path, configuration, cacheConf, BloomType.NONE,
65          NoOpDataBlockEncoder.INSTANCE);
66  
67      StoreFile.Reader reader = storeFile.createReader();
68      StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
69      KeyValue current;
70  
71      scanner.seek(KeyValue.LOWESTKEY);
72      while (null != (current = scanner.next())) {
73        allKeyValues.add(current);
74      }
75  
76      storeFile.closeReader(cacheConf.shouldEvictOnClose());
77  
78      // pick seeks by random
79      List<KeyValue> seeks = new ArrayList<KeyValue>();
80      for (int i = 0; i < numberOfSeeks; ++i) {
81        KeyValue keyValue = allKeyValues.get(
82            randomizer.nextInt(allKeyValues.size()));
83        seeks.add(keyValue);
84      }
85  
86      clearBlockCache();
87  
88      return seeks;
89    }
90  
91    private void runTest(Path path, HFileDataBlockEncoder blockEncoder,
92        List<KeyValue> seeks) throws IOException {
93      // read all of the key values
94      StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
95          path, configuration, cacheConf, BloomType.NONE, blockEncoder);
96  
97      long totalSize = 0;
98  
99      StoreFile.Reader reader = storeFile.createReader();
100     StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
101 
102     long startReadingTime = System.nanoTime();
103     KeyValue current;
104     scanner.seek(KeyValue.LOWESTKEY);
105     while (null != (current = scanner.next())) { // just iterate it!
106       if (current.getLength() < 0) {
107         throw new IOException("Negative KV size: " + current);
108       }
109       totalSize += current.getLength();
110     }
111     long finishReadingTime = System.nanoTime();
112 
113     // do seeks
114     long startSeeksTime = System.nanoTime();
115     for (KeyValue keyValue : seeks) {
116       scanner.seek(keyValue);
117       KeyValue toVerify = scanner.next();
118       if (!keyValue.equals(toVerify)) {
119         System.out.println(String.format("KeyValue doesn't match:\n" +
120             "Orig key: %s\n" +
121             "Ret key:  %s", keyValue.getKeyString(), toVerify.getKeyString()));
122         break;
123       }
124     }
125     long finishSeeksTime = System.nanoTime();
126     if (finishSeeksTime < startSeeksTime) {
127       throw new AssertionError("Finish time " + finishSeeksTime +
128           " is earlier than start time " + startSeeksTime);
129     }
130 
131     // write some stats
132     double readInMbPerSec = (totalSize * NANOSEC_IN_SEC) /
133         (BYTES_IN_MEGABYTES * (finishReadingTime - startReadingTime));
134     double seeksPerSec = (seeks.size() * NANOSEC_IN_SEC) /
135         (finishSeeksTime - startSeeksTime);
136 
137     storeFile.closeReader(cacheConf.shouldEvictOnClose());
138     clearBlockCache();
139 
140     System.out.println(blockEncoder);
141     System.out.printf("  Read speed:       %8.2f (MB/s)\n", readInMbPerSec);
142     System.out.printf("  Seeks per second: %8.2f (#/s)\n", seeksPerSec);
143     System.out.printf("  Total KV size:    %d\n", totalSize);
144   }
145 
146   /**
147    * @param path Path to the HFile which will be used.
148    * @param encoders List of encoders which will be used for tests.
149    * @throws IOException if there is a bug while reading from disk
150    */
151   public void runTests(Path path, List<HFileDataBlockEncoder> encoders)
152       throws IOException {
153     List<KeyValue> seeks = prepareListOfTestSeeks(path);
154 
155     for (HFileDataBlockEncoder blockEncoder : encoders) {
156       runTest(path, blockEncoder, seeks);
157     }
158   }
159 
160   /**
161    * Command line interface:
162    * @param args Takes one argument - file size.
163    * @throws IOException if there is a bug while reading from disk
164    */
165   public static void main(final String[] args) throws IOException {
166     if (args.length < 1) {
167       printUsage();
168       System.exit(-1);
169     }
170 
171     Path path = new Path(args[0]);
172     List<HFileDataBlockEncoder> encoders =
173         new ArrayList<HFileDataBlockEncoder>();
174 
175     for (DataBlockEncoding encodingAlgo : DataBlockEncoding.values()) {
176       encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE,
177           encodingAlgo));
178     }
179 
180     EncodedSeekPerformanceTest utility = new EncodedSeekPerformanceTest();
181     utility.runTests(path, encoders);
182 
183     System.exit(0);
184   }
185 
186   private static void printUsage() {
187     System.out.println("Usage: one argument, name of the HFile");
188   }
189 
190   private void clearBlockCache() {
191     ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
192   }
193 }