View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import java.io.IOException;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Random;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configured;
30  import org.apache.hadoop.fs.FileStatus;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.HBaseConfiguration;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.KeyValue;
41  import org.apache.hadoop.hbase.KeyValueUtil;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.client.Put;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.util.Tool;
48  import org.apache.hadoop.util.ToolRunner;
49  
50  /**
51   * This class runs performance benchmarks for {@link HLog}.
52   * See usage for this tool by running:
53   * <code>$ hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation -h</code>
54   */
55  @InterfaceAudience.Private
56  public final class HLogPerformanceEvaluation extends Configured implements Tool {
57    static final Log LOG = LogFactory.getLog(HLogPerformanceEvaluation.class.getName());
58  
59    private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
60  
61    static final String TABLE_NAME = "HLogPerformanceEvaluation";
62    static final String QUALIFIER_PREFIX = "q";
63    static final String FAMILY_PREFIX = "cf";
64  
65    private int numQualifiers = 1;
66    private int valueSize = 512;
67    private int keySize = 16;
68  
69    /**
70     * Perform HLog.append() of Put object, for the number of iterations requested.
71     * Keys and Vaues are generated randomly, the number of column familes,
72     * qualifiers and key/value size is tunable by the user.
73     */
74    class HLogPutBenchmark implements Runnable {
75      private final long numIterations;
76      private final int numFamilies;
77      private final boolean noSync;
78      private final HRegion region;
79      private final HTableDescriptor htd;
80  
81      HLogPutBenchmark(final HRegion region, final HTableDescriptor htd,
82          final long numIterations, final boolean noSync) {
83        this.numIterations = numIterations;
84        this.noSync = noSync;
85        this.numFamilies = htd.getColumnFamilies().length;
86        this.region = region;
87        this.htd = htd;
88      }
89  
90      public void run() {
91        byte[] key = new byte[keySize];
92        byte[] value = new byte[valueSize];
93        Random rand = new Random(Thread.currentThread().getId());
94        HLog hlog = region.getLog();
95  
96        try {
97          long startTime = System.currentTimeMillis();
98          for (int i = 0; i < numIterations; ++i) {
99            Put put = setupPut(rand, key, value, numFamilies);
100           long now = System.currentTimeMillis();
101           WALEdit walEdit = new WALEdit();
102           addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
103           HRegionInfo hri = region.getRegionInfo();
104           if (this.noSync) {
105             hlog.appendNoSync(hri, hri.getTableName(), walEdit,
106                               HConstants.DEFAULT_CLUSTER_ID, now, htd);
107           } else {
108             hlog.append(hri, hri.getTableName(), walEdit, now, htd);
109           }
110         }
111         long totalTime = (System.currentTimeMillis() - startTime);
112         logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
113       } catch (Exception e) {
114         LOG.error(getClass().getSimpleName() + " Thread failed", e);
115       }
116     }
117   }
118 
119   @Override
120   public int run(String[] args) throws Exception {
121     Path rootRegionDir = null;
122     int numThreads = 1;
123     long numIterations = 10000;
124     int numFamilies = 1;
125     boolean noSync = false;
126     boolean verify = false;
127     boolean verbose = false;
128     long roll = Long.MAX_VALUE;
129     // Process command line args
130     for (int i = 0; i < args.length; i++) {
131       String cmd = args[i];
132       try {
133         if (cmd.equals("-threads")) {
134           numThreads = Integer.parseInt(args[++i]);
135         } else if (cmd.equals("-iterations")) {
136           numIterations = Long.parseLong(args[++i]);
137         } else if (cmd.equals("-path")) {
138           rootRegionDir = new Path(args[++i]);
139         } else if (cmd.equals("-families")) {
140           numFamilies = Integer.parseInt(args[++i]);
141         } else if (cmd.equals("-qualifiers")) {
142           numQualifiers = Integer.parseInt(args[++i]);
143         } else if (cmd.equals("-keySize")) {
144           keySize = Integer.parseInt(args[++i]);
145         } else if (cmd.equals("-valueSize")) {
146           valueSize = Integer.parseInt(args[++i]);
147         } else if (cmd.equals("-nosync")) {
148           noSync = true;
149         } else if (cmd.equals("-verify")) {
150           verify = true;
151         } else if (cmd.equals("-verbose")) {
152           verbose = true;
153         } else if (cmd.equals("-roll")) {
154           roll = Long.parseLong(args[++i]);
155         } else if (cmd.equals("-h")) {
156           printUsageAndExit();
157         } else if (cmd.equals("--help")) {
158           printUsageAndExit();
159         } else {
160           System.err.println("UNEXPECTED: " + cmd);
161           printUsageAndExit();
162         }
163       } catch (Exception e) {
164         printUsageAndExit();
165       }
166     }
167 
168     // Run HLog Performance Evaluation
169     FileSystem fs = FileSystem.get(getConf());
170     LOG.info("" + fs);
171     try {
172       if (rootRegionDir == null) {
173         rootRegionDir = TEST_UTIL.getDataTestDir("HLogPerformanceEvaluation");
174       }
175       rootRegionDir = rootRegionDir.makeQualified(fs);
176       cleanRegionRootDir(fs, rootRegionDir);
177       // Initialize Table Descriptor
178       HTableDescriptor htd = createHTableDescriptor(numFamilies);
179       final long whenToRoll = roll;
180       HLog hlog = new FSHLog(fs, rootRegionDir, "wals", getConf()) {
181         int appends = 0;
182         protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
183             HTableDescriptor htd)
184         throws IOException {
185           this.appends++;
186           if (this.appends % whenToRoll == 0) {
187             LOG.info("Rolling after " + appends + " edits");
188             rollWriter();
189           }
190           super.doWrite(info, logKey, logEdit, htd);
191         };
192       };
193       hlog.rollWriter();
194       HRegion region = null;
195       try {
196         region = openRegion(fs, rootRegionDir, htd, hlog);
197         long putTime = runBenchmark(new HLogPutBenchmark(region, htd, numIterations, noSync), numThreads);
198         logBenchmarkResult("Summary: threads=" + numThreads + ", iterations=" + numIterations,
199           numIterations * numThreads, putTime);
200         if (region != null) {
201           closeRegion(region);
202           region = null;
203         }
204         if (verify) {
205           Path dir = ((FSHLog) hlog).getDir();
206           long editCount = 0;
207           for (FileStatus fss: fs.listStatus(dir)) {
208             editCount += verify(fss.getPath(), verbose);
209           }
210           long expected = numIterations * numThreads;
211           if (editCount != expected) {
212             throw new IllegalStateException("Counted=" + editCount + ", expected=" + expected);
213           }
214         }
215       } finally {
216         if (region != null) closeRegion(region);
217         // Remove the root dir for this test region
218         cleanRegionRootDir(fs, rootRegionDir);
219       }
220     } finally {
221       fs.close();
222     }
223 
224     return(0);
225   }
226 
227   private static HTableDescriptor createHTableDescriptor(final int numFamilies) {
228     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE_NAME));
229     for (int i = 0; i < numFamilies; ++i) {
230       HColumnDescriptor colDef = new HColumnDescriptor(FAMILY_PREFIX + i);
231       htd.addFamily(colDef);
232     }
233     return htd;
234   }
235 
236   /**
237    * Verify the content of the WAL file.
238    * Verify that sequenceids are ascending and that the file has expected number
239    * of edits.
240    * @param wal
241    * @return Count of edits.
242    * @throws IOException
243    */
244   private long verify(final Path wal, final boolean verbose) throws IOException {
245     HLog.Reader reader = HLogFactory.createReader(wal.getFileSystem(getConf()), 
246         wal, getConf());
247     long previousSeqid = -1;
248     long count = 0;
249     try {
250       while (true) {
251         Entry e = reader.next();
252         if (e == null) break;
253         count++;
254         long seqid = e.getKey().getLogSeqNum();
255         if (verbose) LOG.info("seqid=" + seqid);
256         if (previousSeqid >= seqid) {
257           throw new IllegalStateException("wal=" + wal.getName() +
258             ", previousSeqid=" + previousSeqid + ", seqid=" + seqid);
259         }
260         previousSeqid = seqid;
261       }
262     } finally {
263       reader.close();
264     }
265     return count;
266   }
267 
268   private static void logBenchmarkResult(String testName, long numTests, long totalTime) {
269     float tsec = totalTime / 1000.0f;
270     LOG.info(String.format("%s took %.3fs %.3fops/s", testName, tsec, numTests / tsec));
271   }
272 
273   private void printUsageAndExit() {
274     System.err.printf("Usage: bin/hbase %s [options]\n", getClass().getName());
275     System.err.println(" where [options] are:");
276     System.err.println("  -h|-help         Show this help and exit.");
277     System.err.println("  -threads <N>     Number of threads writing on the WAL.");
278     System.err.println("  -iterations <N>  Number of iterations per thread.");
279     System.err.println("  -path <PATH>     Path where region's root directory is created.");
280     System.err.println("  -families <N>    Number of column families to write.");
281     System.err.println("  -qualifiers <N>  Number of qualifiers to write.");
282     System.err.println("  -keySize <N>     Row key size in byte.");
283     System.err.println("  -valueSize <N>   Row/Col value size in byte.");
284     System.err.println("  -nosync          Append without syncing");
285     System.err.println("  -verify          Verify edits written in sequence");
286     System.err.println("  -verbose         Output extra info; e.g. all edit seq ids when verifying");
287     System.err.println("  -roll <N>        Roll the way every N appends");
288     System.err.println("");
289     System.err.println("Examples:");
290     System.err.println("");
291     System.err.println(" To run 100 threads on hdfs with log rolling every 10k edits and verification afterward do:");
292     System.err.println(" $ ./bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLogPerformanceEvaluation \\");
293     System.err.println("    -conf ./core-site.xml -path hdfs://example.org:7000/tmp -threads 100 -roll 10000 -verify");
294     System.exit(1);
295   }
296 
297   private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final HLog hlog)
298   throws IOException {
299     // Initialize HRegion
300     HRegionInfo regionInfo = new HRegionInfo(htd.getTableName());
301     return HRegion.createHRegion(regionInfo, dir, getConf(), htd, hlog);
302   }
303 
304   private void closeRegion(final HRegion region) throws IOException {
305     if (region != null) {
306       region.close();
307       HLog wal = region.getLog();
308       if (wal != null) wal.close();
309     }
310   }
311 
312   private void cleanRegionRootDir(final FileSystem fs, final Path dir) throws IOException {
313     if (fs.exists(dir)) {
314       fs.delete(dir, true);
315     }
316   }
317 
318   private Put setupPut(Random rand, byte[] key, byte[] value, final int numFamilies) {
319     rand.nextBytes(key);
320     Put put = new Put(key);
321     for (int cf = 0; cf < numFamilies; ++cf) {
322       for (int q = 0; q < numQualifiers; ++q) {
323         rand.nextBytes(value);
324         put.add(Bytes.toBytes(FAMILY_PREFIX + cf), Bytes.toBytes(QUALIFIER_PREFIX + q), value);
325       }
326     }
327     return put;
328   }
329 
330   private void addFamilyMapToWALEdit(Map<byte[], List<Cell>> familyMap,
331       WALEdit walEdit) {
332     for (List<Cell> edits : familyMap.values()) {
333       for (Cell cell : edits) {
334         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
335         walEdit.add(kv);
336       }
337     }
338   }
339 
340   private long runBenchmark(Runnable runnable, final int numThreads) throws InterruptedException {
341     Thread[] threads = new Thread[numThreads];
342     long startTime = System.currentTimeMillis();
343     for (int i = 0; i < numThreads; ++i) {
344       threads[i] = new Thread(runnable);
345       threads[i].start();
346     }
347     for (Thread t : threads) t.join();
348     long endTime = System.currentTimeMillis();
349     return(endTime - startTime);
350   }
351 
352   /**
353    * The guts of the {@link #main} method.
354    * Call this method to avoid the {@link #main(String[])} System.exit.
355    * @param args
356    * @return errCode
357    * @throws Exception 
358    */
359   static int innerMain(final String [] args) throws Exception {
360     return ToolRunner.run(HBaseConfiguration.create(), new HLogPerformanceEvaluation(), args);
361   }
362 
363   public static void main(String[] args) throws Exception {
364      System.exit(innerMain(args));
365   }
366 }