1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.BufferedReader;
22 import java.io.FileReader;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.Random;
28
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.LocalFileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.fs.RawLocalFileSystem;
33 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
34 import org.apache.hadoop.hbase.util.Bytes;
35
36
37
38
39 public class RandomSeek {
40 private static List<String> slurp(String fname) throws IOException {
41 BufferedReader istream = new BufferedReader(new FileReader(fname));
42 String str;
43 List<String> l = new ArrayList<String>();
44 while ( (str=istream.readLine()) != null) {
45 String [] parts = str.split(",");
46 l.add(parts[0] + ":" + parts[1] + ":" + parts[2]);
47 }
48 istream.close();
49 return l;
50 }
51
52 private static String randKey(List<String> keys) {
53 Random r = new Random();
54
55 return "2" + Integer.toString(7+r.nextInt(2)) + Integer.toString(r.nextInt(100));
56
57 }
58
59 public static void main(String [] argv) throws IOException {
60 Configuration conf = new Configuration();
61 conf.setInt("io.file.buffer.size", 64*1024);
62 RawLocalFileSystem rlfs = new RawLocalFileSystem();
63 rlfs.setConf(conf);
64 LocalFileSystem lfs = new LocalFileSystem(rlfs);
65
66 Path path = new Path("/Users/ryan/rfile.big.txt");
67 long start = System.currentTimeMillis();
68 SimpleBlockCache cache = new SimpleBlockCache();
69 CacheConfig cacheConf = new CacheConfig(cache, true, false, false, false,
70 false, false, false, true);
71
72 Reader reader = HFile.createReader(lfs, path, cacheConf, conf);
73 reader.loadFileInfo();
74 System.out.println(reader.getTrailer());
75 long end = System.currentTimeMillis();
76
77 System.out.println("Index read time: " + (end - start));
78
79 List<String> keys = slurp("/Users/ryan/xaa.50k");
80
81
82 HFileScanner scanner = reader.getScanner(false, true);
83 int count;
84 long totalBytes = 0;
85 int notFound = 0;
86
87 start = System.nanoTime();
88 for(count = 0; count < 500000; ++count) {
89 String key = randKey(keys);
90 byte [] bkey = Bytes.toBytes(key);
91 int res = scanner.seekTo(bkey);
92 if (res == 0) {
93 ByteBuffer k = scanner.getKey();
94 ByteBuffer v = scanner.getValue();
95 totalBytes += k.limit();
96 totalBytes += v.limit();
97 } else {
98 ++ notFound;
99 }
100 if (res == -1) {
101 scanner.seekTo();
102 }
103
104 for (int i = 0; i < 1000; ++i) {
105 if (!scanner.next())
106 break;
107 ByteBuffer k = scanner.getKey();
108 ByteBuffer v = scanner.getValue();
109 totalBytes += k.limit();
110 totalBytes += v.limit();
111 }
112
113 if ( count % 1000 == 0 ) {
114 end = System.nanoTime();
115
116 System.out.println("Cache block count: " + cache.size() + " dumped: "+ cache.dumps);
117
118 double msTime = ((end - start) / 1000000.0);
119 System.out.println("Seeked: "+ count + " in " + msTime + " (ms) "
120 + (1000.0 / msTime ) + " seeks/ms "
121 + (msTime / 1000.0) + " ms/seek");
122
123 start = System.nanoTime();
124 }
125 }
126 System.out.println("Total bytes: " + totalBytes + " not found: " + notFound);
127 }
128 }