1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import java.io.IOException;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Random;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configured;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.Cell;
34 import org.apache.hadoop.hbase.HBaseConfiguration;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.KeyValue;
41 import org.apache.hadoop.hbase.KeyValueUtil;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.client.Put;
44 import org.apache.hadoop.hbase.regionserver.HRegion;
45 import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.util.Tool;
48 import org.apache.hadoop.util.ToolRunner;
49
50
51
52
53
54
55 @InterfaceAudience.Private
56 public final class HLogPerformanceEvaluation extends Configured implements Tool {
57 static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName());
58
59 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
60
61 static final String TABLE_NAME = "HLogPerformanceEvaluation";
62 static final String QUALIFIER_PREFIX = "q";
63 static final String FAMILY_PREFIX = "cf";
64
65 private int numQualifiers = 1;
66 private int valueSize = 512;
67 private int keySize = 16;
68
69
70
71
72
73
74 class HLogPutBenchmark implements Runnable {
75 private final long numIterations;
76 private final int numFamilies;
77 private final boolean noSync;
78 private final HRegion region;
79 private final HTableDescriptor htd;
80
81 HLogPutBenchmark(final HRegion region, final HTableDescriptor htd,
82 final long numIterations, final boolean noSync) {
83 this.numIterations = numIterations;
84 this.noSync = noSync;
85 this.numFamilies = htd.getColumnFamilies().length;
86 this.region = region;
87 this.htd = htd;
88 }
89
90 public void run() {
91 byte[] key = new byte[keySize];
92 byte[] value = new byte[valueSize];
93 Random rand = new Random(Thread.currentThread().getId());
94 HLog hlog = region.getLog();
95
96 try {
97 long startTime = System.currentTimeMillis();
98 for (int i = 0; i < numIterations; ++i) {
99 Put put = setupPut(rand, key, value, numFamilies);
100 long now = System.currentTimeMillis();
101 WALEdit walEdit = new WALEdit();
102 addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
103 HRegionInfo hri = region.getRegionInfo();
104 if (this.noSync) {
105 hlog.appendNoSync(hri, hri.getTableName(), walEdit,
106 HConstants.DEFAULT_CLUSTER_ID, now, htd);
107 } else {
108 hlog.append(hri, hri.getTableName(), walEdit, now, htd);
109 }
110 }
111 long totalTime = (System.currentTimeMillis() - startTime);
112 logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
113 } catch (Exception e) {
114 LOG.error(getClass().getSimpleName() + " Thread failed", e);
115 }
116 }
117 }
118
119 @Override
120 public int run(String[] args) throws Exception {
121 Path rootRegionDir = null;
122 int numThreads = 1;
123 long numIterations = 10000;
124 int numFamilies = 1;
125 boolean noSync = false;
126 boolean verify = false;
127 boolean verbose = false;
128 long roll = Long.MAX_VALUE;
129
130 for (int i = 0; i < args.length; i++) {
131 String cmd = args[i];
132 try {
133 if (cmd.equals("-threads")) {
134 numThreads = Integer.parseInt(args[++i]);
135 } else if (cmd.equals("-iterations")) {
136 numIterations = Long.parseLong(args[++i]);
137 } else if (cmd.equals("-path")) {
138 rootRegionDir = new Path(args[++i]);
139 } else if (cmd.equals("-families")) {
140 numFamilies = Integer.parseInt(args[++i]);
141 } else if (cmd.equals("-qualifiers")) {
142 numQualifiers = Integer.parseInt(args[++i]);
143 } else if (cmd.equals("-keySize")) {
144 keySize = Integer.parseInt(args[++i]);
145 } else if (cmd.equals("-valueSize")) {
146 valueSize = Integer.parseInt(args[++i]);
147 } else if (cmd.equals("-nosync")) {
148 noSync = true;
149 } else if (cmd.equals("-verify")) {
150 verify = true;
151 } else if (cmd.equals("-verbose")) {
152 verbose = true;
153 } else if (cmd.equals("-roll")) {
154 roll = Long.parseLong(args[++i]);
155 } else if (cmd.equals("-h")) {
156 printUsageAndExit();
157 } else if (cmd.equals("--help")) {
158 printUsageAndExit();
159 } else {
160 System.err.println("UNEXPECTED: " + cmd);
161 printUsageAndExit();
162 }
163 } catch (Exception e) {
164 printUsageAndExit();
165 }
166 }
167
168
169 FileSystem fs = FileSystem.get(getConf());
170 LOG.info("" + fs);
171 try {
172 if (rootRegionDir == null) {
173 rootRegionDir = TEST_UTIL.getDataTestDir("HLogPerformanceEvaluation");
174 }
175 rootRegionDir = rootRegionDir.makeQualified(fs);
176 cleanRegionRootDir(fs, rootRegionDir);
177
178 HTableDescriptor htd = createHTableDescriptor(numFamilies);
179 final long whenToRoll = roll;
180 HLog hlog = new FSHLog(fs, rootRegionDir, "wals", getConf()) {
181 int appends = 0;
182 protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
183 HTableDescriptor htd)
184 throws IOException {
185 this.appends++;
186 if (this.appends % whenToRoll == 0) {
187 LOG.info("Rolling after " + appends + " edits");
188 rollWriter();
189 }
190 super.doWrite(info, logKey, logEdit, htd);
191 };
192 };
193 hlog.rollWriter();
194 HRegion region = null;
195 try {
196 region = openRegion(fs, rootRegionDir, htd, hlog);
197 long putTime = runBenchmark(new HLogPutBenchmark(region, htd, numIterations, noSync), numThreads);
198 logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations,
199 numIterations * numThreads, putTime);
200 if (region != null) {
201 closeRegion(region);
202 region = null;
203 }
204 if (verify) {
205 Path dir = ((FSHLog) hlog).getDir();
206 long editCount = 0;
207 for (FileStatus fss: fs.listStatus(dir)) {
208 editCount += verify(fss.getPath(), verbose);
209 }
210 long expected = numIterations * numThreads;
211 if (editCount != expected) {
212 throw new IllegalStateException("Counted=" + editCount + ", expected=" + expected);
213 }
214 }
215 } finally {
216 if (region != null) closeRegion(region);
217
218 cleanRegionRootDir(fs, rootRegionDir);
219 }
220 } finally {
221 fs.close();
222 }
223
224 return(0);
225 }
226
227 private static HTableDescriptor createHTableDescriptor(final int numFamilies) {
228 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE_NAME));
229 for (int i = 0; i < numFamilies; ++i) {
230 HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i);
231 htd.addFamily(colDef);
232 }
233 return htd;
234 }
235
236
237
238
239
240
241
242
243
244 private long verify(final Path wal, final boolean verbose) throws IOException {
245 HLog.Reader reader = HLogFactory.createReader(wal.getFileSystem(getConf()),
246 wal, getConf());
247 long previousSeqid = -1;
248 long count = 0;
249 try {
250 while (true) {
251 Entry e = reader.next();
252 if (e == null) break;
253 count++;
254 long seqid = e.getKey().getLogSeqNum();
255 if (verbose) LOG.info("seqid=" + seqid);
256 if (previousSeqid >= seqid) {
257 throw new IllegalStateException("wal=" + wal.getName() +
258 ", previousSeqid=" + previousSeqid + ", seqid=" + seqid);
259 }
260 previousSeqid = seqid;
261 }
262 } finally {
263 reader.close();
264 }
265 return count;
266 }
267
268 private static void logBenchmarkResult(String testName, long numTests, long totalTime) {
269 float tsec = totalTime / 1000.0f;
270 LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec));
271 }
272
273 private void printUsageAndExit() {
274 System.err.printf("Usage: bin/hbase %s [options]\n", getClass().getName());
275 System.err.println(" where [options] are:");
276 System.err.println(" -h|-help Show this help and exit.");
277 System.err.println(" -threads <N> Number of threads writing on the WAL.");
278 System.err.println(" -iterations <N> Number of iterations per thread.");
279 System.err.println(" -path <PATH> Path where region's root directory is created.");
280 System.err.println(" -families <N> Number of column families to write.");
281 System.err.println(" -qualifiers <N> Number of qualifiers to write.");
282 System.err.println(" -keySize <N> Row key size in byte.");
283 System.err.println(" -valueSize <N> Row/Col value size in byte.");
284 System.err.println(" -nosync Append without syncing");
285 System.err.println(" -verify Verify edits written in sequence");
286 System.err.println(" -verbose Output extra info; e.g. all edit seq ids when verifying");
287 System.err.println(" -roll <N> Roll the way every N appends");
288 System.err.println("");
289 System.err.println("Examples:");
290 System.err.println("");
291 System.err.println(" To run 100 threads on hdfs with log rolling every 10k edits and verification afterward do:");
292 System.err.println(" $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation \\");
293 System.err.println(" -conf ./core-site.xml -path hdfs://example.org:7000/tmp -threads 100 -roll 10000 -verify");
294 System.exit(1);
295 }
296
297 private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final HLog hlog)
298 throws IOException {
299
300 HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
301 return HRegion.createHRegion(regionInfo, dir, getConf(), htd, hlog);
302 }
303
304 private void closeRegion(final HRegion region) throws IOException {
305 if (region != null) {
306 region.close();
307 HLog wal = region.getLog();
308 if (wal != null) wal.close();
309 }
310 }
311
312 private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException {
313 if (fs.exists(dir)) {
314 fs.delete(dir, true);
315 }
316 }
317
318 private Put setupPut(Random rand, byte[] key, byte[] value, final int numFamilies) {
319 rand.nextBytes(key);
320 Put put = new Put(key);
321 for (int cf = 0; cf < numFamilies; ++cf) {
322 for (int q = 0; q < numQualifiers; ++q) {
323 rand.nextBytes(value);
324 put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value);
325 }
326 }
327 return put;
328 }
329
330 private void addFamilyMapToWALEdit(Map<byte[], List<Cell>> familyMap,
331 WALEdit walEdit) {
332 for (List<Cell> edits : familyMap.values()) {
333 for (Cell cell : edits) {
334 KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
335 walEdit.add(kv);
336 }
337 }
338 }
339
340 private long runBenchmark(Runnable runnable, final int numThreads) throws InterruptedException {
341 Thread[] threads = new Thread[numThreads];
342 long startTime = System.currentTimeMillis();
343 for (int i = 0; i < numThreads; ++i) {
344 threads[i] = new Thread(runnable);
345 threads[i].start();
346 }
347 for (Thread t : threads) t.join();
348 long endTime = System.currentTimeMillis();
349 return(endTime - startTime);
350 }
351
352
353
354
355
356
357
358
359 static int innerMain(final String [] args) throws Exception {
360 return ToolRunner.run(HBaseConfiguration.create(), new HLogPerformanceEvaluation(), args);
361 }
362
363 public static void main(String[] args) throws Exception {
364 System.exit(innerMain(args));
365 }
366 }