1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.Random;
26 import java.util.UUID;
27 import java.util.concurrent.TimeUnit;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.classification.InterfaceAudience;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.conf.Configured;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.Cell;
38 import org.apache.hadoop.hbase.HBaseConfiguration;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.KeyValue;
44 import org.apache.hadoop.hbase.KeyValueUtil;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.client.Put;
47 import org.apache.hadoop.hbase.regionserver.HRegion;
48 import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.util.Tool;
52 import org.apache.hadoop.util.ToolRunner;
53
54 import com.yammer.metrics.core.Meter;
55 import com.yammer.metrics.core.MetricsRegistry;
56 import com.yammer.metrics.reporting.ConsoleReporter;
57
58
59
60
61
62
63 @InterfaceAudience.Private
64 public final class HLogPerformanceEvaluation extends Configured implements Tool {
65 static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName());
66 private final MetricsRegistry metrics = new MetricsRegistry();
67 private final Meter syncMeter =
68 metrics.newMeter(HLogPerformanceEvaluation.class, "syncMeter", "syncs", TimeUnit.MILLISECONDS);
69 private final Meter appendMeter =
70 metrics.newMeter(HLogPerformanceEvaluation.class, "append", "bytes", TimeUnit.MILLISECONDS);
71
72 private HBaseTestingUtility TEST_UTIL;
73
74 static final String TABLE_NAME = "HLogPerformanceEvaluation";
75 static final String QUALIFIER_PREFIX = "q";
76 static final String FAMILY_PREFIX = "cf";
77
78 private int numQualifiers = 1;
79 private int valueSize = 512;
80 private int keySize = 16;
81
82 @Override
83 public void setConf(Configuration conf) {
84 super.setConf(conf);
85 TEST_UTIL = new HBaseTestingUtility(conf);
86 }
87
88
89
90
91
92
93 class HLogPutBenchmark implements Runnable {
94 private final long numIterations;
95 private final int numFamilies;
96 private final boolean noSync;
97 private final HRegion region;
98 private final int syncInterval;
99 private final HTableDescriptor htd;
100
101 HLogPutBenchmark(final HRegion region, final HTableDescriptor htd,
102 final long numIterations, final boolean noSync, final int syncInterval) {
103 this.numIterations = numIterations;
104 this.noSync = noSync;
105 this.syncInterval = syncInterval;
106 this.numFamilies = htd.getColumnFamilies().length;
107 this.region = region;
108 this.htd = htd;
109 }
110
111 @Override
112 public void run() {
113 byte[] key = new byte[keySize];
114 byte[] value = new byte[valueSize];
115 Random rand = new Random(Thread.currentThread().getId());
116 HLog hlog = region.getLog();
117
118 try {
119 long startTime = System.currentTimeMillis();
120 int lastSync = 0;
121 for (int i = 0; i < numIterations; ++i) {
122 Put put = setupPut(rand, key, value, numFamilies);
123 long now = System.currentTimeMillis();
124 WALEdit walEdit = new WALEdit();
125 addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
126 HRegionInfo hri = region.getRegionInfo();
127 hlog.appendNoSync(hri, hri.getTable(), walEdit, new ArrayList<UUID>(), now, htd);
128 if (!this.noSync) {
129 if (++lastSync >= this.syncInterval) {
130 hlog.sync();
131 lastSync = 0;
132 }
133 }
134 }
135 long totalTime = (System.currentTimeMillis() - startTime);
136 logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
137 } catch (Exception e) {
138 LOG.error(getClass().getSimpleName() + " Thread failed", e);
139 }
140 }
141 }
142
143 @Override
144 public int run(String[] args) throws Exception {
145 Path rootRegionDir = null;
146 int numThreads = 1;
147 long numIterations = 1000000;
148 int numFamilies = 1;
149 int syncInterval = 0;
150 boolean noSync = false;
151 boolean verify = false;
152 boolean verbose = false;
153 boolean cleanup = true;
154 boolean noclosefs = false;
155 long roll = Long.MAX_VALUE;
156
157 for (int i = 0; i < args.length; i++) {
158 String cmd = args[i];
159 try {
160 if (cmd.equals("-threads")) {
161 numThreads = Integer.parseInt(args[++i]);
162 } else if (cmd.equals("-iterations")) {
163 numIterations = Long.parseLong(args[++i]);
164 } else if (cmd.equals("-path")) {
165 rootRegionDir = new Path(args[++i]);
166 } else if (cmd.equals("-families")) {
167 numFamilies = Integer.parseInt(args[++i]);
168 } else if (cmd.equals("-qualifiers")) {
169 numQualifiers = Integer.parseInt(args[++i]);
170 } else if (cmd.equals("-keySize")) {
171 keySize = Integer.parseInt(args[++i]);
172 } else if (cmd.equals("-valueSize")) {
173 valueSize = Integer.parseInt(args[++i]);
174 } else if (cmd.equals("-syncInterval")) {
175 syncInterval = Integer.parseInt(args[++i]);
176 } else if (cmd.equals("-nosync")) {
177 noSync = true;
178 } else if (cmd.equals("-verify")) {
179 verify = true;
180 } else if (cmd.equals("-verbose")) {
181 verbose = true;
182 } else if (cmd.equals("-nocleanup")) {
183 cleanup = false;
184 } else if (cmd.equals("-noclosefs")) {
185 noclosefs = true;
186 } else if (cmd.equals("-roll")) {
187 roll = Long.parseLong(args[++i]);
188 } else if (cmd.equals("-h")) {
189 printUsageAndExit();
190 } else if (cmd.equals("--help")) {
191 printUsageAndExit();
192 } else {
193 System.err.println("UNEXPECTED: " + cmd);
194 printUsageAndExit();
195 }
196 } catch (Exception e) {
197 printUsageAndExit();
198 }
199 }
200
201
202
203 FSUtils.setFsDefault(getConf(), FSUtils.getRootDir(getConf()));
204 FileSystem fs = FileSystem.get(getConf());
205 LOG.info("FileSystem: " + fs);
206 try {
207 if (rootRegionDir == null) {
208 rootRegionDir = TEST_UTIL.getDataTestDirOnTestFS("HLogPerformanceEvaluation");
209 }
210 rootRegionDir = rootRegionDir.makeQualified(fs);
211 cleanRegionRootDir(fs, rootRegionDir);
212
213 HTableDescriptor htd = createHTableDescriptor(numFamilies);
214 final long whenToRoll = roll;
215 HLog hlog = new FSHLog(fs, rootRegionDir, "wals", getConf()) {
216 int appends = 0;
217 @Override
218 protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
219 HTableDescriptor htd)
220 throws IOException {
221 this.appends++;
222 if (this.appends % whenToRoll == 0) {
223 LOG.info("Rolling after " + appends + " edits");
224 rollWriter();
225 }
226 super.doWrite(info, logKey, logEdit, htd);
227 };
228
229 @Override
230 public void postSync() {
231 super.postSync();
232 syncMeter.mark();
233 }
234
235 @Override
236 public void postAppend(List<Entry> entries) {
237 super.postAppend(entries);
238 int size = 0;
239 for (Entry e: entries) size += e.getEdit().heapSize();
240 appendMeter.mark(size);
241 }
242 };
243 hlog.rollWriter();
244 HRegion region = null;
245 try {
246 region = openRegion(fs, rootRegionDir, htd, hlog);
247 ConsoleReporter.enable(this.metrics, 1, TimeUnit.SECONDS);
248 long putTime =
249 runBenchmark(new HLogPutBenchmark(region, htd, numIterations, noSync, syncInterval),
250 numThreads);
251 logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations +
252 ", syncInterval=" + syncInterval, numIterations * numThreads, putTime);
253
254 if (region != null) {
255 closeRegion(region);
256 region = null;
257 }
258 if (verify) {
259 Path dir = ((FSHLog) hlog).getDir();
260 long editCount = 0;
261 FileStatus [] fsss = fs.listStatus(dir);
262 if (fsss.length == 0) throw new IllegalStateException("No WAL found");
263 for (FileStatus fss: fsss) {
264 Path p = fss.getPath();
265 if (!fs.exists(p)) throw new IllegalStateException(p.toString());
266 editCount += verify(p, verbose);
267 }
268 long expected = numIterations * numThreads;
269 if (editCount != expected) {
270 throw new IllegalStateException("Counted=" + editCount + ", expected=" + expected);
271 }
272 }
273 } finally {
274 if (region != null) closeRegion(region);
275
276 if (cleanup) cleanRegionRootDir(fs, rootRegionDir);
277 }
278 } finally {
279
280 if (!noclosefs) fs.close();
281 }
282
283 return(0);
284 }
285
286 private static HTableDescriptor createHTableDescriptor(final int numFamilies) {
287 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE_NAME));
288 for (int i = 0; i < numFamilies; ++i) {
289 HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i);
290 htd.addFamily(colDef);
291 }
292 return htd;
293 }
294
295
296
297
298
299
300
301
302
303 private long verify(final Path wal, final boolean verbose) throws IOException {
304 HLog.Reader reader = HLogFactory.createReader(wal.getFileSystem(getConf()), wal, getConf());
305 long previousSeqid = -1;
306 long count = 0;
307 try {
308 while (true) {
309 Entry e = reader.next();
310 if (e == null) {
311 LOG.debug("Read count=" + count + " from " + wal);
312 break;
313 }
314 count++;
315 long seqid = e.getKey().getLogSeqNum();
316 if (verbose) LOG.info("seqid=" + seqid);
317 if (previousSeqid >= seqid) {
318 throw new IllegalStateException("wal=" + wal.getName() +
319 ", previousSeqid=" + previousSeqid + ", seqid=" + seqid);
320 }
321 previousSeqid = seqid;
322 }
323 } finally {
324 reader.close();
325 }
326 return count;
327 }
328
329 private static void logBenchmarkResult(String testName, long numTests, long totalTime) {
330 float tsec = totalTime / 1000.0f;
331 LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec));
332
333 }
334
335 private void printUsageAndExit() {
336 System.err.printf("Usage: bin/hbase %s [options]\n", getClass().getName());
337 System.err.println(" where [options] are:");
338 System.err.println(" -h|-help Show this help and exit.");
339 System.err.println(" -threads <N> Number of threads writing on the WAL.");
340 System.err.println(" -iterations <N> Number of iterations per thread.");
341 System.err.println(" -path <PATH> Path where region's root directory is created.");
342 System.err.println(" -families <N> Number of column families to write.");
343 System.err.println(" -qualifiers <N> Number of qualifiers to write.");
344 System.err.println(" -keySize <N> Row key size in byte.");
345 System.err.println(" -valueSize <N> Row/Col value size in byte.");
346 System.err.println(" -nocleanup Do NOT remove test data when done.");
347 System.err.println(" -noclosefs Do NOT close the filesystem when done.");
348 System.err.println(" -nosync Append without syncing");
349 System.err.println(" -syncInterval <N> Append N edits and then sync. Default=0, i.e. sync every edit.");
350 System.err.println(" -verify Verify edits written in sequence");
351 System.err.println(" -verbose Output extra info; e.g. all edit seq ids when verifying");
352 System.err.println(" -roll <N> Roll the way every N appends");
353 System.err.println("");
354 System.err.println("Examples:");
355 System.err.println("");
356 System.err.println(" To run 100 threads on hdfs with log rolling every 10k edits and verification afterward do:");
357 System.err.println(" $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation \\");
358 System.err.println(" -conf ./core-site.xml -path hdfs://example.org:7000/tmp -threads 100 -roll 10000 -verify");
359 System.exit(1);
360 }
361
362 private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final HLog hlog)
363 throws IOException {
364
365 HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
366 return HRegion.createHRegion(regionInfo, dir, getConf(), htd, hlog);
367 }
368
369 private void closeRegion(final HRegion region) throws IOException {
370 if (region != null) {
371 region.close();
372 HLog wal = region.getLog();
373 if (wal != null) wal.close();
374 }
375 }
376
377 private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException {
378 if (fs.exists(dir)) {
379 fs.delete(dir, true);
380 }
381 }
382
383 private Put setupPut(Random rand, byte[] key, byte[] value, final int numFamilies) {
384 rand.nextBytes(key);
385 Put put = new Put(key);
386 for (int cf = 0; cf < numFamilies; ++cf) {
387 for (int q = 0; q < numQualifiers; ++q) {
388 rand.nextBytes(value);
389 put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value);
390 }
391 }
392 return put;
393 }
394
395 private void addFamilyMapToWALEdit(Map<byte[], List<Cell>> familyMap,
396 WALEdit walEdit) {
397 for (List<Cell> edits : familyMap.values()) {
398 for (Cell cell : edits) {
399 KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
400 walEdit.add(kv);
401 }
402 }
403 }
404
405 private long runBenchmark(Runnable runnable, final int numThreads) throws InterruptedException {
406 Thread[] threads = new Thread[numThreads];
407 long startTime = System.currentTimeMillis();
408 for (int i = 0; i < numThreads; ++i) {
409 threads[i] = new Thread(runnable, "t" + i);
410 threads[i].start();
411 }
412 for (Thread t : threads) t.join();
413 long endTime = System.currentTimeMillis();
414 return(endTime - startTime);
415 }
416
417
418
419
420
421
422
423
424 static int innerMain(final Configuration c, final String [] args) throws Exception {
425 return ToolRunner.run(c, new HLogPerformanceEvaluation(), args);
426 }
427
428 public static void main(String[] args) throws Exception {
429 System.exit(innerMain(HBaseConfiguration.create(), args));
430 }
431 }