1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.regionserver;
18
19 import java.io.IOException;
20 import java.util.ArrayList;
21 import java.util.List;
22 import java.util.Random;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.HBaseTestingUtility;
27 import org.apache.hadoop.hbase.HConstants;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
30 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
31 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
32 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
33 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
34 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
35
36
37
38
39
40 public class EncodedSeekPerformanceTest {
41 private static final double NANOSEC_IN_SEC = 1000.0 * 1000.0 * 1000.0;
42 private static final double BYTES_IN_MEGABYTES = 1024.0 * 1024.0;
43
44 public static int DEFAULT_NUMBER_OF_SEEKS = 10000;
45
46 private final HBaseTestingUtility testingUtility = new HBaseTestingUtility();
47 private Configuration configuration = testingUtility.getConfiguration();
48 private CacheConfig cacheConf = new CacheConfig(configuration);
49 private Random randomizer;
50 private int numberOfSeeks;
51
52
53 public EncodedSeekPerformanceTest() {
54 configuration.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f);
55 randomizer = new Random(42l);
56 numberOfSeeks = DEFAULT_NUMBER_OF_SEEKS;
57 }
58
59 private List<KeyValue> prepareListOfTestSeeks(Path path) throws IOException {
60 List<KeyValue> allKeyValues = new ArrayList<KeyValue>();
61
62
63 StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
64 path, configuration, cacheConf, BloomType.NONE,
65 NoOpDataBlockEncoder.INSTANCE);
66
67 StoreFile.Reader reader = storeFile.createReader();
68 StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
69 KeyValue current;
70
71 scanner.seek(KeyValue.LOWESTKEY);
72 while (null != (current = scanner.next())) {
73 allKeyValues.add(current);
74 }
75
76 storeFile.closeReader(cacheConf.shouldEvictOnClose());
77
78
79 List<KeyValue> seeks = new ArrayList<KeyValue>();
80 for (int i = 0; i < numberOfSeeks; ++i) {
81 KeyValue keyValue = allKeyValues.get(
82 randomizer.nextInt(allKeyValues.size()));
83 seeks.add(keyValue);
84 }
85
86 clearBlockCache();
87
88 return seeks;
89 }
90
91 private void runTest(Path path, HFileDataBlockEncoder blockEncoder,
92 List<KeyValue> seeks) throws IOException {
93
94 StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
95 path, configuration, cacheConf, BloomType.NONE, blockEncoder);
96
97 long totalSize = 0;
98
99 StoreFile.Reader reader = storeFile.createReader();
100 StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
101
102 long startReadingTime = System.nanoTime();
103 KeyValue current;
104 scanner.seek(KeyValue.LOWESTKEY);
105 while (null != (current = scanner.next())) {
106 if (current.getLength() < 0) {
107 throw new IOException("Negative KV size: " + current);
108 }
109 totalSize += current.getLength();
110 }
111 long finishReadingTime = System.nanoTime();
112
113
114 long startSeeksTime = System.nanoTime();
115 for (KeyValue keyValue : seeks) {
116 scanner.seek(keyValue);
117 KeyValue toVerify = scanner.next();
118 if (!keyValue.equals(toVerify)) {
119 System.out.println(String.format("KeyValue doesn't match:\n" +
120 "Orig key: %s\n" +
121 "Ret key: %s", keyValue.getKeyString(), toVerify.getKeyString()));
122 break;
123 }
124 }
125 long finishSeeksTime = System.nanoTime();
126 if (finishSeeksTime < startSeeksTime) {
127 throw new AssertionError("Finish time " + finishSeeksTime +
128 " is earlier than start time " + startSeeksTime);
129 }
130
131
132 double readInMbPerSec = (totalSize * NANOSEC_IN_SEC) /
133 (BYTES_IN_MEGABYTES * (finishReadingTime - startReadingTime));
134 double seeksPerSec = (seeks.size() * NANOSEC_IN_SEC) /
135 (finishSeeksTime - startSeeksTime);
136
137 storeFile.closeReader(cacheConf.shouldEvictOnClose());
138 clearBlockCache();
139
140 System.out.println(blockEncoder);
141 System.out.printf(" Read speed: %8.2f (MB/s)\n", readInMbPerSec);
142 System.out.printf(" Seeks per second: %8.2f (#/s)\n", seeksPerSec);
143 System.out.printf(" Total KV size: %d\n", totalSize);
144 }
145
146
147
148
149
150
151 public void runTests(Path path, List<HFileDataBlockEncoder> encoders)
152 throws IOException {
153 List<KeyValue> seeks = prepareListOfTestSeeks(path);
154
155 for (HFileDataBlockEncoder blockEncoder : encoders) {
156 runTest(path, blockEncoder, seeks);
157 }
158 }
159
160
161
162
163
164
165 public static void main(final String[] args) throws IOException {
166 if (args.length < 1) {
167 printUsage();
168 System.exit(-1);
169 }
170
171 Path path = new Path(args[0]);
172 List<HFileDataBlockEncoder> encoders =
173 new ArrayList<HFileDataBlockEncoder>();
174
175 for (DataBlockEncoding encodingAlgo : DataBlockEncoding.values()) {
176 encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE,
177 encodingAlgo));
178 }
179
180 EncodedSeekPerformanceTest utility = new EncodedSeekPerformanceTest();
181 utility.runTests(path, encoders);
182
183 System.exit(0);
184 }
185
186 private static void printUsage() {
187 System.out.println("Usage: one argument, name of the HFile");
188 }
189
190 private void clearBlockCache() {
191 ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
192 }
193 }