1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import java.util.Map;
22  import java.util.List;
23  import java.util.Random;
24  import java.io.IOException;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  
29  import org.apache.hadoop.fs.FileStatus;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.util.Tool;
33  import org.apache.hadoop.util.ToolRunner;
34  import org.apache.hadoop.conf.Configured;
35  
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HBaseConfiguration;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.KeyValue;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.client.Put;
45  import org.apache.hadoop.hbase.regionserver.HRegion;
46  import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
47  
48  /**
49   * This class runs performance benchmarks for {@link HLog}.
50   * See usage for this tool by running:
51   * <code>$ hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation -h</code>
52   */
53  public final class HLogPerformanceEvaluation extends Configured implements Tool {
54    static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName());
55  
56    private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57  
58    static final String TABLE_NAME = "HLogPerformanceEvaluation";
59    static final String QUALIFIER_PREFIX = "q";
60    static final String FAMILY_PREFIX = "cf";
61  
62    private int numQualifiers = 1;
63    private int valueSize = 512;
64    private int keySize = 16;
65  
66    /**
67     * Perform HLog.append() of Put object, for the number of iterations requested.
68     * Keys and Vaues are generated randomly, the number of column familes,
69     * qualifiers and key/value size is tunable by the user.
70     */
71    class HLogPutBenchmark implements Runnable {
72      private final long numIterations;
73      private final int numFamilies;
74      private final boolean noSync;
75      private final HRegion region;
76      private final HTableDescriptor htd;
77  
78      HLogPutBenchmark(final HRegion region, final HTableDescriptor htd,
79          final long numIterations, final boolean noSync) {
80        this.numIterations = numIterations;
81        this.noSync = noSync;
82        this.numFamilies = htd.getColumnFamilies().length;
83        this.region = region;
84        this.htd = htd;
85      }
86  
87      public void run() {
88        byte[] key = new byte[keySize];
89        byte[] value = new byte[valueSize];
90        Random rand = new Random(Thread.currentThread().getId());
91        HLog hlog = region.getLog();
92  
93        try {
94          long startTime = System.currentTimeMillis();
95          for (int i = 0; i < numIterations; ++i) {
96            Put put = setupPut(rand, key, value, numFamilies);
97            long now = System.currentTimeMillis();
98            WALEdit walEdit = new WALEdit();
99            addFamilyMapToWALEdit(put.getFamilyMap(), walEdit);
100           HRegionInfo hri = region.getRegionInfo();
101           if (this.noSync) {
102             hlog.appendNoSync(hri, hri.getTableName(), walEdit,
103                               HConstants.DEFAULT_CLUSTER_ID, now, htd);
104           } else {
105             hlog.append(hri, hri.getTableName(), walEdit, now, htd);
106           }
107         }
108         long totalTime = (System.currentTimeMillis() - startTime);
109         logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
110       } catch (Exception e) {
111         LOG.error(getClass().getSimpleName() + " Thread failed", e);
112       }
113     }
114   }
115 
116   @Override
117   public int run(String[] args) throws Exception {
118     Path rootRegionDir = null;
119     int numThreads = 1;
120     long numIterations = 10000;
121     int numFamilies = 1;
122     boolean noSync = false;
123     boolean verify = false;
124     boolean verbose = false;
125     long roll = Long.MAX_VALUE;
126     // Process command line args
127     for (int i = 0; i < args.length; i++) {
128       String cmd = args[i];
129       try {
130         if (cmd.equals("-threads")) {
131           numThreads = Integer.parseInt(args[++i]);
132         } else if (cmd.equals("-iterations")) {
133           numIterations = Long.parseLong(args[++i]);
134         } else if (cmd.equals("-path")) {
135           rootRegionDir = new Path(args[++i]);
136         } else if (cmd.equals("-families")) {
137           numFamilies = Integer.parseInt(args[++i]);
138         } else if (cmd.equals("-qualifiers")) {
139           numQualifiers = Integer.parseInt(args[++i]);
140         } else if (cmd.equals("-keySize")) {
141           keySize = Integer.parseInt(args[++i]);
142         } else if (cmd.equals("-valueSize")) {
143           valueSize = Integer.parseInt(args[++i]);
144         } else if (cmd.equals("-nosync")) {
145           noSync = true;
146         } else if (cmd.equals("-verify")) {
147           verify = true;
148         } else if (cmd.equals("-verbose")) {
149           verbose = true;
150         } else if (cmd.equals("-roll")) {
151           roll = Long.parseLong(args[++i]);
152         } else if (cmd.equals("-h")) {
153           printUsageAndExit();
154         } else if (cmd.equals("--help")) {
155           printUsageAndExit();
156         } else {
157           System.err.println("UNEXPECTED: " + cmd);
158           printUsageAndExit();
159         }
160       } catch (Exception e) {
161         printUsageAndExit();
162       }
163     }
164 
165     // Run HLog Performance Evaluation
166     FileSystem fs = FileSystem.get(getConf());
167     LOG.info("" + fs);
168     try {
169       if (rootRegionDir == null) {
170         rootRegionDir = TEST_UTIL.getDataTestDir("HLogPerformanceEvaluation");
171       }
172       rootRegionDir = rootRegionDir.makeQualified(fs);
173       cleanRegionRootDir(fs, rootRegionDir);
174       // Initialize Table Descriptor
175       HTableDescriptor htd = createHTableDescriptor(numFamilies);
176       final long whenToRoll = roll;
177       HLog hlog = new HLog(fs, new Path(rootRegionDir, "wals"),
178           new Path(rootRegionDir, "old.wals"), getConf()) {
179         int appends = 0;
180         protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
181             HTableDescriptor htd)
182         throws IOException {
183           this.appends++;
184           if (this.appends % whenToRoll == 0) {
185             LOG.info("Rolling after " + appends + " edits");
186             rollWriter();
187           }
188           super.doWrite(info, logKey, logEdit, htd);
189         };
190       };
191       hlog.rollWriter();
192       HRegion region = null;
193       try {
194         region = openRegion(fs, rootRegionDir, htd, hlog);
195         long putTime = runBenchmark(new HLogPutBenchmark(region, htd, numIterations, noSync), numThreads);
196         logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations,
197           numIterations * numThreads, putTime);
198         if (region != null) {
199           closeRegion(region);
200           region = null;
201         }
202         if (verify) {
203           Path dir = hlog.getDir();
204           long editCount = 0;
205           for (FileStatus fss: fs.listStatus(dir)) {
206             editCount += verify(fss.getPath(), verbose);
207           }
208           long expected = numIterations * numThreads;
209           if (editCount != expected) {
210             throw new IllegalStateException("Counted=" + editCount + ", expected=" + expected);
211           }
212         }
213       } finally {
214         if (region != null) closeRegion(region);
215         // Remove the root dir for this test region
216         cleanRegionRootDir(fs, rootRegionDir);
217       }
218     } finally {
219       fs.close();
220     }
221 
222     return(0);
223   }
224 
225   private static HTableDescriptor createHTableDescriptor(final int numFamilies) {
226     HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
227     for (int i = 0; i < numFamilies; ++i) {
228       HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i);
229       htd.addFamily(colDef);
230     }
231     return htd;
232   }
233 
234   /**
235    * Verify the content of the WAL file.
236    * Verify that sequenceids are ascending and that the file has expected number
237    * of edits.
238    * @param wal
239    * @return Count of edits.
240    * @throws IOException
241    */
242   private long verify(final Path wal, final boolean verbose) throws IOException {
243     HLog.Reader reader = HLog.getReader(wal.getFileSystem(getConf()), wal, getConf());
244     long previousSeqid = -1;
245     long count = 0;
246     try {
247       while (true) {
248         Entry e = reader.next();
249         if (e == null) break;
250         count++;
251         long seqid = e.getKey().getLogSeqNum();
252         if (verbose) LOG.info("seqid=" + seqid);
253         if (previousSeqid >= seqid) {
254           throw new IllegalStateException("wal=" + wal.getName() +
255             ", previousSeqid=" + previousSeqid + ", seqid=" + seqid);
256         }
257         previousSeqid = seqid;
258       }
259     } finally {
260       reader.close();
261     }
262     return count;
263   }
264 
265   private static void logBenchmarkResult(String testName, long numTests, long totalTime) {
266     float tsec = totalTime / 1000.0f;
267     LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec));
268   }
269 
270   private void printUsageAndExit() {
271     System.err.printf("Usage: bin/hbase %s [options]\n", getClass().getName());
272     System.err.println(" where [options] are:");
273     System.err.println("  -h|-help         Show this help and exit.");
274     System.err.println("  -threads <N>     Number of threads writing on the WAL.");
275     System.err.println("  -iterations <N>  Number of iterations per thread.");
276     System.err.println("  -path <PATH>     Path where region's root directory is created.");
277     System.err.println("  -families <N>    Number of column families to write.");
278     System.err.println("  -qualifiers <N>  Number of qualifiers to write.");
279     System.err.println("  -keySize <N>     Row key size in byte.");
280     System.err.println("  -valueSize <N>   Row/Col value size in byte.");
281     System.err.println("  -nosync          Append without syncing");
282     System.err.println("  -verify          Verify edits written in sequence");
283     System.err.println("  -verbose         Output extra info; e.g. all edit seq ids when verifying");
284     System.err.println("  -roll <N>        Roll the way every N appends");
285     System.err.println("");
286     System.err.println("Examples:");
287     System.err.println("");
288     System.err.println(" To run 100 threads on hdfs with log rolling every 10k edits and verification afterward do:");
289     System.err.println(" $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation \\");
290     System.err.println("    -conf ./core-site.xml -path hdfs://example.org:7000/tmp -threads 100 -roll 10000 -verify");
291     System.exit(1);
292   }
293 
294   private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final HLog hlog)
295   throws IOException {
296     // Initialize HRegion
297     HRegionInfo regionInfo = new HRegionInfo(htd.getName());
298     return HRegion.createHRegion(regionInfo, dir, getConf(), htd, hlog);
299   }
300 
301   private void closeRegion(final HRegion region) throws IOException {
302     if (region != null) {
303       region.close();
304       HLog wal = region.getLog();
305       if (wal != null) wal.close();
306     }
307   }
308 
309   private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException {
310     if (fs.exists(dir)) {
311       fs.delete(dir, true);
312     }
313   }
314 
315   private Put setupPut(Random rand, byte[] key, byte[] value, final int numFamilies) {
316     rand.nextBytes(key);
317     Put put = new Put(key);
318     for (int cf = 0; cf < numFamilies; ++cf) {
319       for (int q = 0; q < numQualifiers; ++q) {
320         rand.nextBytes(value);
321         put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value);
322       }
323     }
324     return put;
325   }
326 
327   private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap, WALEdit walEdit) {
328     for (List<KeyValue> edits : familyMap.values()) {
329       for (KeyValue kv : edits) {
330         walEdit.add(kv);
331       }
332     }
333   }
334 
335   private long runBenchmark(Runnable runnable, final int numThreads) throws InterruptedException {
336     Thread[] threads = new Thread[numThreads];
337     long startTime = System.currentTimeMillis();
338     for (int i = 0; i < numThreads; ++i) {
339       threads[i] = new Thread(runnable);
340       threads[i].start();
341     }
342     for (Thread t : threads) t.join();
343     long endTime = System.currentTimeMillis();
344     return(endTime - startTime);
345   }
346 
347   /**
348    * The guts of the {@link #main} method.
349    * Call this method to avoid the {@link #main(String[])} System.exit.
350    * @param args
351    * @return errCode
352    * @throws Exception 
353    */
354   static int innerMain(final String [] args) throws Exception {
355     return ToolRunner.run(HBaseConfiguration.create(), new HLogPerformanceEvaluation(), args);
356   }
357 
358   public static void main(String[] args) throws Exception {
359      System.exit(innerMain(args));
360   }
361 }