View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.BufferedReader;
22  import java.io.FileReader;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Random;
28  
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.LocalFileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.fs.RawLocalFileSystem;
33  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
34  import org.apache.hadoop.hbase.util.Bytes;
35  
36  /**
37   * Random seek test.
38   */
39  public class RandomSeek {
40    private static List<String> slurp(String fname) throws IOException {
41      BufferedReader istream = new BufferedReader(new FileReader(fname));
42      String str;
43      List<String> l = new ArrayList<String>();
44      while ( (str=istream.readLine()) != null) {
45        String [] parts = str.split(",");
46        l.add(parts[0] + ":" + parts[1] + ":" + parts[2]);
47      }
48      istream.close();
49      return l;
50    }
51  
52    private static String randKey(List<String> keys) {
53      Random r = new Random();
54      //return keys.get(r.nextInt(keys.size()));
55      return "2" + Integer.toString(7+r.nextInt(2)) + Integer.toString(r.nextInt(100));
56      //return new String(r.nextInt(100));
57    }
58  
59    public static void main(String [] argv) throws IOException {
60      Configuration conf = new Configuration();
61      conf.setInt("io.file.buffer.size", 64*1024);
62      RawLocalFileSystem rlfs = new RawLocalFileSystem();
63      rlfs.setConf(conf);
64      LocalFileSystem lfs = new LocalFileSystem(rlfs);
65  
66      Path path = new Path("/Users/ryan/rfile.big.txt");
67      long start = System.currentTimeMillis();
68      SimpleBlockCache cache = new SimpleBlockCache();
69      CacheConfig cacheConf = new CacheConfig(cache, true, false, false, false,
70          false, false, false, true);
71  
72      Reader reader = HFile.createReader(lfs, path, cacheConf, conf);
73      reader.loadFileInfo();
74      System.out.println(reader.getTrailer());
75      long end = System.currentTimeMillis();
76  
77      System.out.println("Index read time: " + (end - start));
78  
79      List<String> keys = slurp("/Users/ryan/xaa.50k");
80  
81      // Get a scanner that doesn't cache and that uses pread.
82      HFileScanner scanner = reader.getScanner(false, true);
83      int count;
84      long totalBytes = 0;
85      int notFound = 0;
86  
87      start = System.nanoTime();
88      for(count = 0; count < 500000; ++count) {
89        String key = randKey(keys);
90        byte [] bkey = Bytes.toBytes(key);
91        int res = scanner.seekTo(bkey);
92        if (res == 0) {
93          ByteBuffer k = scanner.getKey();
94          ByteBuffer v = scanner.getValue();
95          totalBytes += k.limit();
96          totalBytes += v.limit();
97        } else {
98          ++ notFound;
99        }
100       if (res == -1) {
101         scanner.seekTo();
102       }
103       // Scan for another 1000 rows.
104       for (int i = 0; i < 1000; ++i) {
105         if (!scanner.next())
106           break;
107         ByteBuffer k = scanner.getKey();
108         ByteBuffer v = scanner.getValue();
109         totalBytes += k.limit();
110         totalBytes += v.limit();
111       }
112 
113       if ( count % 1000 == 0 ) {
114         end = System.nanoTime();
115 
116             System.out.println("Cache block count: " + cache.size() + " dumped: "+ cache.dumps);
117             //System.out.println("Cache size: " + cache.heapSize());
118             double msTime = ((end - start) / 1000000.0);
119             System.out.println("Seeked: "+ count + " in " + msTime + " (ms) "
120                 + (1000.0 / msTime ) + " seeks/ms "
121                 + (msTime / 1000.0) + " ms/seek");
122 
123             start = System.nanoTime();
124       }
125     }
126     System.out.println("Total bytes: " + totalBytes + " not found: " + notFound);
127   }
128 }