1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.regionserver;
18
19 import java.io.IOException;
20 import java.util.ArrayList;
21 import java.util.List;
22 import java.util.Random;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.HBaseTestingUtility;
27 import org.apache.hadoop.hbase.HConstants;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
30 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
31 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
32 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
33 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
34 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
35 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
36
37
38
39
40
41 public class EncodedSeekPerformanceTest {
42 private static final double NANOSEC_IN_SEC = 1000.0 * 1000.0 * 1000.0;
43 private static final double BYTES_IN_MEGABYTES = 1024.0 * 1024.0;
44
45 public static int DEFAULT_NUMBER_OF_SEEKS = 10000;
46
47 private final HBaseTestingUtility testingUtility = new HBaseTestingUtility();
48 private Configuration configuration = testingUtility.getConfiguration();
49 private CacheConfig cacheConf = new CacheConfig(configuration);
50 private Random randomizer;
51 private int numberOfSeeks;
52
53
54 public EncodedSeekPerformanceTest() {
55 configuration.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f);
56 randomizer = new Random(42l);
57 numberOfSeeks = DEFAULT_NUMBER_OF_SEEKS;
58 }
59
60 private List<KeyValue> prepareListOfTestSeeks(Path path) throws IOException {
61 List<KeyValue> allKeyValues = new ArrayList<KeyValue>();
62
63
64 StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
65 path, configuration, cacheConf, BloomType.NONE,
66 NoOpDataBlockEncoder.INSTANCE);
67
68 StoreFile.Reader reader = storeFile.createReader();
69 StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
70 KeyValue current;
71
72 scanner.seek(KeyValue.LOWESTKEY);
73 while (null != (current = scanner.next())) {
74 allKeyValues.add(current);
75 }
76
77 storeFile.closeReader(cacheConf.shouldEvictOnClose());
78
79
80 List<KeyValue> seeks = new ArrayList<KeyValue>();
81 for (int i = 0; i < numberOfSeeks; ++i) {
82 KeyValue keyValue = allKeyValues.get(
83 randomizer.nextInt(allKeyValues.size()));
84 seeks.add(keyValue);
85 }
86
87 clearBlockCache();
88
89 return seeks;
90 }
91
92 private void runTest(Path path, HFileDataBlockEncoder blockEncoder,
93 List<KeyValue> seeks) throws IOException {
94
95 StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
96 path, configuration, cacheConf, BloomType.NONE, blockEncoder);
97
98 long totalSize = 0;
99
100 StoreFile.Reader reader = storeFile.createReader();
101 StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
102
103 long startReadingTime = System.nanoTime();
104 KeyValue current;
105 scanner.seek(KeyValue.LOWESTKEY);
106 while (null != (current = scanner.next())) {
107 if (current.getLength() < 0) {
108 throw new IOException("Negative KV size: " + current);
109 }
110 totalSize += current.getLength();
111 }
112 long finishReadingTime = System.nanoTime();
113
114
115 long startSeeksTime = System.nanoTime();
116 for (KeyValue keyValue : seeks) {
117 scanner.seek(keyValue);
118 KeyValue toVerify = scanner.next();
119 if (!keyValue.equals(toVerify)) {
120 System.out.println(String.format("KeyValue doesn't match:\n" +
121 "Orig key: %s\n" +
122 "Ret key: %s", keyValue.getKeyString(), toVerify.getKeyString()));
123 break;
124 }
125 }
126 long finishSeeksTime = System.nanoTime();
127 if (finishSeeksTime < startSeeksTime) {
128 throw new AssertionError("Finish time " + finishSeeksTime +
129 " is earlier than start time " + startSeeksTime);
130 }
131
132
133 double readInMbPerSec = (totalSize * NANOSEC_IN_SEC) /
134 (BYTES_IN_MEGABYTES * (finishReadingTime - startReadingTime));
135 double seeksPerSec = (seeks.size() * NANOSEC_IN_SEC) /
136 (finishSeeksTime - startSeeksTime);
137
138 storeFile.closeReader(cacheConf.shouldEvictOnClose());
139 clearBlockCache();
140
141 System.out.println(blockEncoder);
142 System.out.printf(" Read speed: %8.2f (MB/s)\n", readInMbPerSec);
143 System.out.printf(" Seeks per second: %8.2f (#/s)\n", seeksPerSec);
144 System.out.printf(" Total KV size: %d\n", totalSize);
145 }
146
147
148
149
150
151
152 public void runTests(Path path, List<HFileDataBlockEncoder> encoders)
153 throws IOException {
154 List<KeyValue> seeks = prepareListOfTestSeeks(path);
155
156 for (HFileDataBlockEncoder blockEncoder : encoders) {
157 runTest(path, blockEncoder, seeks);
158 }
159 }
160
161
162
163
164
165
166 public static void main(final String[] args) throws IOException {
167 if (args.length < 1) {
168 printUsage();
169 System.exit(-1);
170 }
171
172 Path path = new Path(args[0]);
173 List<HFileDataBlockEncoder> encoders =
174 new ArrayList<HFileDataBlockEncoder>();
175
176 encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE));
177 for (DataBlockEncoding encodingAlgo : DataBlockEncoding.values()) {
178 encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE,
179 encodingAlgo));
180 }
181
182 EncodedSeekPerformanceTest utility = new EncodedSeekPerformanceTest();
183 utility.runTests(path, encoders);
184
185 System.exit(0);
186 }
187
188 private static void printUsage() {
189 System.out.println("Usage: one argument, name of the HFile");
190 }
191
192 private void clearBlockCache() {
193 ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
194 }
195 }