1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import java.util.Map;
22 import java.util.List;
23 import java.util.Random;
24 import java.io.IOException;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28
29 import org.apache.hadoop.fs.FileStatus;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.util.Tool;
33 import org.apache.hadoop.util.ToolRunner;
34 import org.apache.hadoop.conf.Configured;
35
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HBaseConfiguration;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.KeyValue;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.client.Put;
45 import org.apache.hadoop.hbase.regionserver.HRegion;
46 import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
47
48
49
50
51
52
53 public final class HLogPerformanceEvaluation extends Configured implements Tool {
54 static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName());
55
56 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57
58 static final String TABLE_NAME = "HLogPerformanceEvaluation";
59 static final String QUALIFIER_PREFIX = "q";
60 static final String FAMILY_PREFIX = "cf";
61
62 private int numQualifiers = 1;
63 private int valueSize = 512;
64 private int keySize = 16;
65
66
67
68
69
70
71 class HLogPutBenchmark implements Runnable {
72 private final long numIterations;
73 private final int numFamilies;
74 private final boolean noSync;
75 private final HRegion region;
76 private final HTableDescriptor htd;
77
78 HLogPutBenchmark(final HRegion region, final HTableDescriptor htd,
79 final long numIterations, final boolean noSync) {
80 this.numIterations = numIterations;
81 this.noSync = noSync;
82 this.numFamilies = htd.getColumnFamilies().length;
83 this.region = region;
84 this.htd = htd;
85 }
86
87 public void run() {
88 byte[] key = new byte[keySize];
89 byte[] value = new byte[valueSize];
90 Random rand = new Random(Thread.currentThread().getId());
91 HLog hlog = region.getLog();
92
93 try {
94 long startTime = System.currentTimeMillis();
95 for (int i = 0; i < numIterations; ++i) {
96 Put put = setupPut(rand, key, value, numFamilies);
97 long now = System.currentTimeMillis();
98 WALEdit walEdit = new WALEdit();
99 addFamilyMapToWALEdit(put.getFamilyMap(), walEdit);
100 HRegionInfo hri = region.getRegionInfo();
101 if (this.noSync) {
102 hlog.appendNoSync(hri, hri.getTableName(), walEdit,
103 HConstants.DEFAULT_CLUSTER_ID, now, htd);
104 } else {
105 hlog.append(hri, hri.getTableName(), walEdit, now, htd);
106 }
107 }
108 long totalTime = (System.currentTimeMillis() - startTime);
109 logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
110 } catch (Exception e) {
111 LOG.error(getClass().getSimpleName() + " Thread failed", e);
112 }
113 }
114 }
115
116 @Override
117 public int run(String[] args) throws Exception {
118 Path rootRegionDir = null;
119 int numThreads = 1;
120 long numIterations = 10000;
121 int numFamilies = 1;
122 boolean noSync = false;
123 boolean verify = false;
124 boolean verbose = false;
125 long roll = Long.MAX_VALUE;
126
127 for (int i = 0; i < args.length; i++) {
128 String cmd = args[i];
129 try {
130 if (cmd.equals("-threads")) {
131 numThreads = Integer.parseInt(args[++i]);
132 } else if (cmd.equals("-iterations")) {
133 numIterations = Long.parseLong(args[++i]);
134 } else if (cmd.equals("-path")) {
135 rootRegionDir = new Path(args[++i]);
136 } else if (cmd.equals("-families")) {
137 numFamilies = Integer.parseInt(args[++i]);
138 } else if (cmd.equals("-qualifiers")) {
139 numQualifiers = Integer.parseInt(args[++i]);
140 } else if (cmd.equals("-keySize")) {
141 keySize = Integer.parseInt(args[++i]);
142 } else if (cmd.equals("-valueSize")) {
143 valueSize = Integer.parseInt(args[++i]);
144 } else if (cmd.equals("-nosync")) {
145 noSync = true;
146 } else if (cmd.equals("-verify")) {
147 verify = true;
148 } else if (cmd.equals("-verbose")) {
149 verbose = true;
150 } else if (cmd.equals("-roll")) {
151 roll = Long.parseLong(args[++i]);
152 } else if (cmd.equals("-h")) {
153 printUsageAndExit();
154 } else if (cmd.equals("--help")) {
155 printUsageAndExit();
156 } else {
157 System.err.println("UNEXPECTED: " + cmd);
158 printUsageAndExit();
159 }
160 } catch (Exception e) {
161 printUsageAndExit();
162 }
163 }
164
165
166 FileSystem fs = FileSystem.get(getConf());
167 LOG.info("" + fs);
168 try {
169 if (rootRegionDir == null) {
170 rootRegionDir = TEST_UTIL.getDataTestDir("HLogPerformanceEvaluation");
171 }
172 rootRegionDir = rootRegionDir.makeQualified(fs);
173 cleanRegionRootDir(fs, rootRegionDir);
174
175 HTableDescriptor htd = createHTableDescriptor(numFamilies);
176 final long whenToRoll = roll;
177 HLog hlog = new HLog(fs, new Path(rootRegionDir, "wals"),
178 new Path(rootRegionDir, "old.wals"), getConf()) {
179 int appends = 0;
180 protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
181 HTableDescriptor htd)
182 throws IOException {
183 this.appends++;
184 if (this.appends % whenToRoll == 0) {
185 LOG.info("Rolling after " + appends + " edits");
186 rollWriter();
187 }
188 super.doWrite(info, logKey, logEdit, htd);
189 };
190 };
191 hlog.rollWriter();
192 HRegion region = null;
193 try {
194 region = openRegion(fs, rootRegionDir, htd, hlog);
195 long putTime = runBenchmark(new HLogPutBenchmark(region, htd, numIterations, noSync), numThreads);
196 logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations,
197 numIterations * numThreads, putTime);
198 if (region != null) {
199 closeRegion(region);
200 region = null;
201 }
202 if (verify) {
203 Path dir = hlog.getDir();
204 long editCount = 0;
205 for (FileStatus fss: fs.listStatus(dir)) {
206 editCount += verify(fss.getPath(), verbose);
207 }
208 long expected = numIterations * numThreads;
209 if (editCount != expected) {
210 throw new IllegalStateException("Counted=" + editCount + ", expected=" + expected);
211 }
212 }
213 } finally {
214 if (region != null) closeRegion(region);
215
216 cleanRegionRootDir(fs, rootRegionDir);
217 }
218 } finally {
219 fs.close();
220 }
221
222 return(0);
223 }
224
225 private static HTableDescriptor createHTableDescriptor(final int numFamilies) {
226 HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
227 for (int i = 0; i < numFamilies; ++i) {
228 HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i);
229 htd.addFamily(colDef);
230 }
231 return htd;
232 }
233
234
235
236
237
238
239
240
241
242 private long verify(final Path wal, final boolean verbose) throws IOException {
243 HLog.Reader reader = HLog.getReader(wal.getFileSystem(getConf()), wal, getConf());
244 long previousSeqid = -1;
245 long count = 0;
246 try {
247 while (true) {
248 Entry e = reader.next();
249 if (e == null) break;
250 count++;
251 long seqid = e.getKey().getLogSeqNum();
252 if (verbose) LOG.info("seqid=" + seqid);
253 if (previousSeqid >= seqid) {
254 throw new IllegalStateException("wal=" + wal.getName() +
255 ", previousSeqid=" + previousSeqid + ", seqid=" + seqid);
256 }
257 previousSeqid = seqid;
258 }
259 } finally {
260 reader.close();
261 }
262 return count;
263 }
264
265 private static void logBenchmarkResult(String testName, long numTests, long totalTime) {
266 float tsec = totalTime / 1000.0f;
267 LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec));
268 }
269
270 private void printUsageAndExit() {
271 System.err.printf("Usage: bin/hbase %s [options]\n", getClass().getName());
272 System.err.println(" where [options] are:");
273 System.err.println(" -h|-help Show this help and exit.");
274 System.err.println(" -threads <N> Number of threads writing on the WAL.");
275 System.err.println(" -iterations <N> Number of iterations per thread.");
276 System.err.println(" -path <PATH> Path where region's root directory is created.");
277 System.err.println(" -families <N> Number of column families to write.");
278 System.err.println(" -qualifiers <N> Number of qualifiers to write.");
279 System.err.println(" -keySize <N> Row key size in byte.");
280 System.err.println(" -valueSize <N> Row/Col value size in byte.");
281 System.err.println(" -nosync Append without syncing");
282 System.err.println(" -verify Verify edits written in sequence");
283 System.err.println(" -verbose Output extra info; e.g. all edit seq ids when verifying");
284 System.err.println(" -roll <N> Roll the way every N appends");
285 System.err.println("");
286 System.err.println("Examples:");
287 System.err.println("");
288 System.err.println(" To run 100 threads on hdfs with log rolling every 10k edits and verification afterward do:");
289 System.err.println(" $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation \\");
290 System.err.println(" -conf ./core-site.xml -path hdfs://example.org:7000/tmp -threads 100 -roll 10000 -verify");
291 System.exit(1);
292 }
293
294 private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final HLog hlog)
295 throws IOException {
296
297 HRegionInfo regionInfo = new HRegionInfo(htd.getName());
298 return HRegion.createHRegion(regionInfo, dir, getConf(), htd, hlog);
299 }
300
301 private void closeRegion(final HRegion region) throws IOException {
302 if (region != null) {
303 region.close();
304 HLog wal = region.getLog();
305 if (wal != null) wal.close();
306 }
307 }
308
309 private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException {
310 if (fs.exists(dir)) {
311 fs.delete(dir, true);
312 }
313 }
314
315 private Put setupPut(Random rand, byte[] key, byte[] value, final int numFamilies) {
316 rand.nextBytes(key);
317 Put put = new Put(key);
318 for (int cf = 0; cf < numFamilies; ++cf) {
319 for (int q = 0; q < numQualifiers; ++q) {
320 rand.nextBytes(value);
321 put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value);
322 }
323 }
324 return put;
325 }
326
327 private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap, WALEdit walEdit) {
328 for (List<KeyValue> edits : familyMap.values()) {
329 for (KeyValue kv : edits) {
330 walEdit.add(kv);
331 }
332 }
333 }
334
335 private long runBenchmark(Runnable runnable, final int numThreads) throws InterruptedException {
336 Thread[] threads = new Thread[numThreads];
337 long startTime = System.currentTimeMillis();
338 for (int i = 0; i < numThreads; ++i) {
339 threads[i] = new Thread(runnable);
340 threads[i].start();
341 }
342 for (Thread t : threads) t.join();
343 long endTime = System.currentTimeMillis();
344 return(endTime - startTime);
345 }
346
347
348
349
350
351
352
353
354 static int innerMain(final String [] args) throws Exception {
355 return ToolRunner.run(HBaseConfiguration.create(), new HLogPerformanceEvaluation(), args);
356 }
357
358 public static void main(String[] args) throws Exception {
359 System.exit(innerMain(args));
360 }
361 }