View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.Random;
26  import java.util.UUID;
27  import java.util.concurrent.TimeUnit;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.classification.InterfaceAudience;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.conf.Configured;
34  import org.apache.hadoop.fs.FileStatus;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.Cell;
38  import org.apache.hadoop.hbase.HBaseConfiguration;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.KeyValue;
44  import org.apache.hadoop.hbase.KeyValueUtil;
45  import org.apache.hadoop.hbase.TableName;
46  import org.apache.hadoop.hbase.client.Put;
47  import org.apache.hadoop.hbase.regionserver.HRegion;
48  import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.FSUtils;
51  import org.apache.hadoop.util.Tool;
52  import org.apache.hadoop.util.ToolRunner;
53  
54  import com.yammer.metrics.core.Meter;
55  import com.yammer.metrics.core.MetricsRegistry;
56  import com.yammer.metrics.reporting.ConsoleReporter;
57  
58  /**
59   * This class runs performance benchmarks for {@link HLog}.
60   * See usage for this tool by running:
61   * <code>$ hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation -h</code>
62   */
63  @InterfaceAudience.Private
64  public final class HLogPerformanceEvaluation extends Configured implements Tool {
65    static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName());
66    private final MetricsRegistry metrics = new MetricsRegistry();
67    private final Meter syncMeter =
68      metrics.newMeter(HLogPerformanceEvaluation.class, "syncMeter", "syncs", TimeUnit.MILLISECONDS);
69    private final Meter appendMeter =
70      metrics.newMeter(HLogPerformanceEvaluation.class, "append", "bytes", TimeUnit.MILLISECONDS);
71  
72    private HBaseTestingUtility TEST_UTIL;
73  
74    static final String TABLE_NAME = "HLogPerformanceEvaluation";
75    static final String QUALIFIER_PREFIX = "q";
76    static final String FAMILY_PREFIX = "cf";
77  
78    private int numQualifiers = 1;
79    private int valueSize = 512;
80    private int keySize = 16;
81  
82    @Override
83    public void setConf(Configuration conf) {
84      super.setConf(conf);
85      TEST_UTIL = new HBaseTestingUtility(conf);
86    }
87  
88    /**
89     * Perform HLog.append() of Put object, for the number of iterations requested.
90     * Keys and Vaues are generated randomly, the number of column families,
91     * qualifiers and key/value size is tunable by the user.
92     */
93    class HLogPutBenchmark implements Runnable {
94      private final long numIterations;
95      private final int numFamilies;
96      private final boolean noSync;
97      private final HRegion region;
98      private final int syncInterval;
99      private final HTableDescriptor htd;
100 
101     HLogPutBenchmark(final HRegion region, final HTableDescriptor htd,
102         final long numIterations, final boolean noSync, final int syncInterval) {
103       this.numIterations = numIterations;
104       this.noSync = noSync;
105       this.syncInterval = syncInterval;
106       this.numFamilies = htd.getColumnFamilies().length;
107       this.region = region;
108       this.htd = htd;
109     }
110 
111     @Override
112     public void run() {
113       byte[] key = new byte[keySize];
114       byte[] value = new byte[valueSize];
115       Random rand = new Random(Thread.currentThread().getId());
116       HLog hlog = region.getLog();
117 
118       try {
119         long startTime = System.currentTimeMillis();
120         int lastSync = 0;
121         for (int i = 0; i < numIterations; ++i) {
122           Put put = setupPut(rand, key, value, numFamilies);
123           long now = System.currentTimeMillis();
124           WALEdit walEdit = new WALEdit();
125           addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
126           HRegionInfo hri = region.getRegionInfo();
127           hlog.appendNoSync(hri, hri.getTable(), walEdit, new ArrayList<UUID>(), now, htd);
128           if (!this.noSync) {
129             if (++lastSync >= this.syncInterval) {
130               hlog.sync();
131               lastSync = 0;
132             }
133           }
134         }
135         long totalTime = (System.currentTimeMillis() - startTime);
136         logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
137       } catch (Exception e) {
138         LOG.error(getClass().getSimpleName() + " Thread failed", e);
139       }
140     }
141   }
142 
143   @Override
144   public int run(String[] args) throws Exception {
145     Path rootRegionDir = null;
146     int numThreads = 1;
147     long numIterations = 1000000;
148     int numFamilies = 1;
149     int syncInterval = 0;
150     boolean noSync = false;
151     boolean verify = false;
152     boolean verbose = false;
153     boolean cleanup = true;
154     boolean noclosefs = false;
155     long roll = Long.MAX_VALUE;
156     // Process command line args
157     for (int i = 0; i < args.length; i++) {
158       String cmd = args[i];
159       try {
160         if (cmd.equals("-threads")) {
161           numThreads = Integer.parseInt(args[++i]);
162         } else if (cmd.equals("-iterations")) {
163           numIterations = Long.parseLong(args[++i]);
164         } else if (cmd.equals("-path")) {
165           rootRegionDir = new Path(args[++i]);
166         } else if (cmd.equals("-families")) {
167           numFamilies = Integer.parseInt(args[++i]);
168         } else if (cmd.equals("-qualifiers")) {
169           numQualifiers = Integer.parseInt(args[++i]);
170         } else if (cmd.equals("-keySize")) {
171           keySize = Integer.parseInt(args[++i]);
172         } else if (cmd.equals("-valueSize")) {
173           valueSize = Integer.parseInt(args[++i]);
174         } else if (cmd.equals("-syncInterval")) {
175           syncInterval = Integer.parseInt(args[++i]);
176         } else if (cmd.equals("-nosync")) {
177           noSync = true;
178         } else if (cmd.equals("-verify")) {
179           verify = true;
180         } else if (cmd.equals("-verbose")) {
181           verbose = true;
182         } else if (cmd.equals("-nocleanup")) {
183           cleanup = false;
184         } else if (cmd.equals("-noclosefs")) {
185           noclosefs = true;
186         } else if (cmd.equals("-roll")) {
187           roll = Long.parseLong(args[++i]);
188         } else if (cmd.equals("-h")) {
189           printUsageAndExit();
190         } else if (cmd.equals("--help")) {
191           printUsageAndExit();
192         } else {
193           System.err.println("UNEXPECTED: " + cmd);
194           printUsageAndExit();
195         }
196       } catch (Exception e) {
197         printUsageAndExit();
198       }
199     }
200 
201     // Run HLog Performance Evaluation
202     // First set the fs from configs.  In case we are on hadoop1
203     FSUtils.setFsDefault(getConf(), FSUtils.getRootDir(getConf()));
204     FileSystem fs = FileSystem.get(getConf());
205     LOG.info("FileSystem: " + fs);
206     try {
207       if (rootRegionDir == null) {
208         rootRegionDir = TEST_UTIL.getDataTestDirOnTestFS("HLogPerformanceEvaluation");
209       }
210       rootRegionDir = rootRegionDir.makeQualified(fs);
211       cleanRegionRootDir(fs, rootRegionDir);
212       // Initialize Table Descriptor
213       HTableDescriptor htd = createHTableDescriptor(numFamilies);
214       final long whenToRoll = roll;
215       HLog hlog = new FSHLog(fs, rootRegionDir, "wals", getConf()) {
216         int appends = 0;
217         @Override
218         protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
219             HTableDescriptor htd)
220         throws IOException {
221           this.appends++;
222           if (this.appends % whenToRoll == 0) {
223             LOG.info("Rolling after " + appends + " edits");
224             rollWriter();
225           }
226           super.doWrite(info, logKey, logEdit, htd);
227         };
228 
229         @Override
230         public void postSync() {
231           super.postSync();
232           syncMeter.mark();
233         }
234 
235         @Override
236         public void postAppend(List<Entry> entries) {
237           super.postAppend(entries);
238           int size = 0;
239           for (Entry e: entries) size += e.getEdit().heapSize();
240           appendMeter.mark(size);
241         }
242       };
243       hlog.rollWriter();
244       HRegion region = null;
245       try {
246         region = openRegion(fs, rootRegionDir, htd, hlog);
247         ConsoleReporter.enable(this.metrics, 1, TimeUnit.SECONDS);
248         long putTime =
249           runBenchmark(new HLogPutBenchmark(region, htd, numIterations, noSync, syncInterval),
250             numThreads);
251         logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations +
252           ", syncInterval=" + syncInterval, numIterations * numThreads, putTime);
253 
254         if (region != null) {
255           closeRegion(region);
256           region = null;
257         }
258         if (verify) {
259           Path dir = ((FSHLog) hlog).getDir();
260           long editCount = 0;
261           FileStatus [] fsss = fs.listStatus(dir);
262           if (fsss.length == 0) throw new IllegalStateException("No WAL found");
263           for (FileStatus fss: fsss) {
264             Path p = fss.getPath();
265             if (!fs.exists(p)) throw new IllegalStateException(p.toString());
266             editCount += verify(p, verbose);
267           }
268           long expected = numIterations * numThreads;
269           if (editCount != expected) {
270             throw new IllegalStateException("Counted=" + editCount + ", expected=" + expected);
271           }
272         }
273       } finally {
274         if (region != null) closeRegion(region);
275         // Remove the root dir for this test region
276         if (cleanup) cleanRegionRootDir(fs, rootRegionDir);
277       }
278     } finally {
279       // We may be called inside a test that wants to keep on using the fs.
280       if (!noclosefs) fs.close();
281     }
282 
283     return(0);
284   }
285 
286   private static HTableDescriptor createHTableDescriptor(final int numFamilies) {
287     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE_NAME));
288     for (int i = 0; i < numFamilies; ++i) {
289       HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i);
290       htd.addFamily(colDef);
291     }
292     return htd;
293   }
294 
295   /**
296    * Verify the content of the WAL file.
297    * Verify that sequenceids are ascending and that the file has expected number
298    * of edits.
299    * @param wal
300    * @return Count of edits.
301    * @throws IOException
302    */
303   private long verify(final Path wal, final boolean verbose) throws IOException {
304     HLog.Reader reader = HLogFactory.createReader(wal.getFileSystem(getConf()), wal, getConf());
305     long previousSeqid = -1;
306     long count = 0;
307     try {
308       while (true) {
309         Entry e = reader.next();
310         if (e == null) {
311           LOG.debug("Read count=" + count + " from " + wal);
312           break;
313         }
314         count++;
315         long seqid = e.getKey().getLogSeqNum();
316         if (verbose) LOG.info("seqid=" + seqid);
317         if (previousSeqid >= seqid) {
318           throw new IllegalStateException("wal=" + wal.getName() +
319             ", previousSeqid=" + previousSeqid + ", seqid=" + seqid);
320         }
321         previousSeqid = seqid;
322       }
323     } finally {
324       reader.close();
325     }
326     return count;
327   }
328 
329   private static void logBenchmarkResult(String testName, long numTests, long totalTime) {
330     float tsec = totalTime / 1000.0f;
331     LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec));
332 
333   }
334 
335   private void printUsageAndExit() {
336     System.err.printf("Usage: bin/hbase %s [options]\n", getClass().getName());
337     System.err.println(" where [options] are:");
338     System.err.println("  -h|-help         Show this help and exit.");
339     System.err.println("  -threads <N>     Number of threads writing on the WAL.");
340     System.err.println("  -iterations <N>  Number of iterations per thread.");
341     System.err.println("  -path <PATH>     Path where region's root directory is created.");
342     System.err.println("  -families <N>    Number of column families to write.");
343     System.err.println("  -qualifiers <N>  Number of qualifiers to write.");
344     System.err.println("  -keySize <N>     Row key size in byte.");
345     System.err.println("  -valueSize <N>   Row/Col value size in byte.");
346     System.err.println("  -nocleanup       Do NOT remove test data when done.");
347     System.err.println("  -noclosefs       Do NOT close the filesystem when done.");
348     System.err.println("  -nosync          Append without syncing");
349     System.err.println("  -syncInterval <N> Append N edits and then sync. Default=0, i.e. sync every edit.");
350     System.err.println("  -verify          Verify edits written in sequence");
351     System.err.println("  -verbose         Output extra info; e.g. all edit seq ids when verifying");
352     System.err.println("  -roll <N>        Roll the way every N appends");
353     System.err.println("");
354     System.err.println("Examples:");
355     System.err.println("");
356     System.err.println(" To run 100 threads on hdfs with log rolling every 10k edits and verification afterward do:");
357     System.err.println(" $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation \\");
358     System.err.println("    -conf ./core-site.xml -path hdfs://example.org:7000/tmp -threads 100 -roll 10000 -verify");
359     System.exit(1);
360   }
361 
362   private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final HLog hlog)
363   throws IOException {
364     // Initialize HRegion
365     HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
366     return HRegion.createHRegion(regionInfo, dir, getConf(), htd, hlog);
367   }
368 
369   private void closeRegion(final HRegion region) throws IOException {
370     if (region != null) {
371       region.close();
372       HLog wal = region.getLog();
373       if (wal != null) wal.close();
374     }
375   }
376 
377   private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException {
378     if (fs.exists(dir)) {
379       fs.delete(dir, true);
380     }
381   }
382 
383   private Put setupPut(Random rand, byte[] key, byte[] value, final int numFamilies) {
384     rand.nextBytes(key);
385     Put put = new Put(key);
386     for (int cf = 0; cf < numFamilies; ++cf) {
387       for (int q = 0; q < numQualifiers; ++q) {
388         rand.nextBytes(value);
389         put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value);
390       }
391     }
392     return put;
393   }
394 
395   private void addFamilyMapToWALEdit(Map<byte[], List<Cell>> familyMap,
396       WALEdit walEdit) {
397     for (List<Cell> edits : familyMap.values()) {
398       for (Cell cell : edits) {
399         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
400         walEdit.add(kv);
401       }
402     }
403   }
404 
405   private long runBenchmark(Runnable runnable, final int numThreads) throws InterruptedException {
406     Thread[] threads = new Thread[numThreads];
407     long startTime = System.currentTimeMillis();
408     for (int i = 0; i < numThreads; ++i) {
409       threads[i] = new Thread(runnable, "t" + i);
410       threads[i].start();
411     }
412     for (Thread t : threads) t.join();
413     long endTime = System.currentTimeMillis();
414     return(endTime - startTime);
415   }
416 
417   /**
418    * The guts of the {@link #main} method.
419    * Call this method to avoid the {@link #main(String[])} System.exit.
420    * @param args
421    * @return errCode
422    * @throws Exception
423    */
424   static int innerMain(final Configuration c, final String [] args) throws Exception {
425     return ToolRunner.run(c, new HLogPerformanceEvaluation(), args);
426   }
427 
428   public static void main(String[] args) throws Exception {
429      System.exit(innerMain(HBaseConfiguration.create(), args));
430   }
431 }