1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Collections;
27 import java.util.Comparator;
28 import java.util.List;
29 import java.util.TreeSet;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestCase;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.io.Reference.Range;
41 import org.apache.hadoop.hbase.io.hfile.HFile;
42 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hdfs.MiniDFSCluster;
45 import org.mockito.Mockito;
46
47 import com.google.common.base.Joiner;
48 import com.google.common.collect.Collections2;
49 import com.google.common.collect.Iterables;
50 import com.google.common.collect.Lists;
51
52
53
54
55 public class TestStoreFile extends HBaseTestCase {
56 static final Log LOG = LogFactory.getLog(TestStoreFile.class);
57 private MiniDFSCluster cluster;
58
59 @Override
60 public void setUp() throws Exception {
61 try {
62 this.cluster = new MiniDFSCluster(this.conf, 2, true, (String[])null);
63
64 this.conf.set(HConstants.HBASE_DIR,
65 this.cluster.getFileSystem().getHomeDirectory().toString());
66 } catch (IOException e) {
67 shutdownDfs(cluster);
68 }
69 super.setUp();
70 }
71
72 @Override
73 public void tearDown() throws Exception {
74 super.tearDown();
75 shutdownDfs(cluster);
76
77
78 }
79
80
81
82
83
84
85 public void testBasicHalfMapFile() throws Exception {
86
87 StoreFile.Writer writer = StoreFile.createWriter(this.fs,
88 new Path(new Path(this.testDir, "regionname"), "familyname"), 2 * 1024);
89 writeStoreFile(writer);
90 checkHalfHFile(new StoreFile(this.fs, writer.getPath(), true, conf,
91 StoreFile.BloomType.NONE, false));
92 }
93
94 private void writeStoreFile(final StoreFile.Writer writer) throws IOException {
95 writeStoreFile(writer, Bytes.toBytes(getName()), Bytes.toBytes(getName()));
96 }
97
98
99
100
101
102
103 public static void writeStoreFile(final StoreFile.Writer writer, byte[] fam, byte[] qualifier)
104 throws IOException {
105 long now = System.currentTimeMillis();
106 try {
107 for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {
108 for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) {
109 byte[] b = new byte[] { (byte) d, (byte) e };
110 writer.append(new KeyValue(b, fam, qualifier, now, b));
111 }
112 }
113 } finally {
114 writer.close();
115 }
116 }
117
118
119
120
121
122
123 public void testReference()
124 throws IOException {
125 Path storedir = new Path(new Path(this.testDir, "regionname"), "familyname");
126 Path dir = new Path(storedir, "1234567890");
127
128 StoreFile.Writer writer = StoreFile.createWriter(this.fs, dir, 8 * 1024);
129 writeStoreFile(writer);
130 StoreFile hsf = new StoreFile(this.fs, writer.getPath(), true, conf,
131 StoreFile.BloomType.NONE, false);
132 StoreFile.Reader reader = hsf.createReader();
133
134
135
136 KeyValue kv = KeyValue.createKeyValueFromKey(reader.midkey());
137 byte [] midRow = kv.getRow();
138 kv = KeyValue.createKeyValueFromKey(reader.getLastKey());
139 byte [] finalRow = kv.getRow();
140
141 Path refPath = StoreFile.split(fs, dir, hsf, midRow, Range.top);
142 StoreFile refHsf = new StoreFile(this.fs, refPath, true, conf,
143 StoreFile.BloomType.NONE, false);
144
145
146 HFileScanner s = refHsf.createReader().getScanner(false, false);
147 for(boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {
148 ByteBuffer bb = s.getKey();
149 kv = KeyValue.createKeyValueFromKey(bb);
150 if (first) {
151 assertTrue(Bytes.equals(kv.getRow(), midRow));
152 first = false;
153 }
154 }
155 assertTrue(Bytes.equals(kv.getRow(), finalRow));
156 }
157
158 private void checkHalfHFile(final StoreFile f)
159 throws IOException {
160 byte [] midkey = f.createReader().midkey();
161 KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
162 byte [] midRow = midKV.getRow();
163
164 Path topDir = Store.getStoreHomedir(this.testDir, "1",
165 Bytes.toBytes(f.getPath().getParent().getName()));
166 if (this.fs.exists(topDir)) {
167 this.fs.delete(topDir, true);
168 }
169 Path topPath = StoreFile.split(this.fs, topDir, f, midRow, Range.top);
170
171 Path bottomDir = Store.getStoreHomedir(this.testDir, "2",
172 Bytes.toBytes(f.getPath().getParent().getName()));
173 if (this.fs.exists(bottomDir)) {
174 this.fs.delete(bottomDir, true);
175 }
176 Path bottomPath = StoreFile.split(this.fs, bottomDir,
177 f, midRow, Range.bottom);
178
179 StoreFile.Reader top = new StoreFile(this.fs, topPath, true, conf,
180 StoreFile.BloomType.NONE, false).createReader();
181 StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath, true, conf,
182 StoreFile.BloomType.NONE, false).createReader();
183 ByteBuffer previous = null;
184 LOG.info("Midkey: " + midKV.toString());
185 ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midkey);
186 try {
187
188
189
190
191 boolean first = true;
192 ByteBuffer key = null;
193 HFileScanner topScanner = top.getScanner(false, false);
194 while ((!topScanner.isSeeked() && topScanner.seekTo()) ||
195 (topScanner.isSeeked() && topScanner.next())) {
196 key = topScanner.getKey();
197
198 assertTrue(topScanner.getReader().getComparator().compare(key.array(),
199 key.arrayOffset(), key.limit(), midkey, 0, midkey.length) >= 0);
200 if (first) {
201 first = false;
202 LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key)));
203 }
204 }
205 LOG.info("Last in top: " + Bytes.toString(Bytes.toBytes(key)));
206
207 first = true;
208 HFileScanner bottomScanner = bottom.getScanner(false, false);
209 while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
210 bottomScanner.next()) {
211 previous = bottomScanner.getKey();
212 key = bottomScanner.getKey();
213 if (first) {
214 first = false;
215 LOG.info("First in bottom: " +
216 Bytes.toString(Bytes.toBytes(previous)));
217 }
218 assertTrue(key.compareTo(bbMidkeyBytes) < 0);
219 }
220 if (previous != null) {
221 LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
222 }
223
224 this.fs.delete(topPath, false);
225 this.fs.delete(bottomPath, false);
226
227
228
229
230 byte [] badmidkey = Bytes.toBytes(" .");
231 topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top);
232 bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
233 Range.bottom);
234 top = new StoreFile(this.fs, topPath, true, conf,
235 StoreFile.BloomType.NONE, false).createReader();
236 bottom = new StoreFile(this.fs, bottomPath, true, conf,
237 StoreFile.BloomType.NONE, false).createReader();
238 bottomScanner = bottom.getScanner(false, false);
239 int count = 0;
240 while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
241 bottomScanner.next()) {
242 count++;
243 }
244
245 assertTrue(count == 0);
246
247 first = true;
248 topScanner = top.getScanner(false, false);
249 while ((!topScanner.isSeeked() && topScanner.seekTo()) ||
250 topScanner.next()) {
251 key = topScanner.getKey();
252 assertTrue(topScanner.getReader().getComparator().compare(key.array(),
253 key.arrayOffset(), key.limit(), badmidkey, 0, badmidkey.length) >= 0);
254 if (first) {
255 first = false;
256 KeyValue keyKV = KeyValue.createKeyValueFromKey(key);
257 LOG.info("First top when key < bottom: " + keyKV);
258 String tmp = Bytes.toString(keyKV.getRow());
259 for (int i = 0; i < tmp.length(); i++) {
260 assertTrue(tmp.charAt(i) == 'a');
261 }
262 }
263 }
264 KeyValue keyKV = KeyValue.createKeyValueFromKey(key);
265 LOG.info("Last top when key < bottom: " + keyKV);
266 String tmp = Bytes.toString(keyKV.getRow());
267 for (int i = 0; i < tmp.length(); i++) {
268 assertTrue(tmp.charAt(i) == 'z');
269 }
270
271 this.fs.delete(topPath, false);
272 this.fs.delete(bottomPath, false);
273
274
275 badmidkey = Bytes.toBytes("|||");
276 topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top);
277 bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
278 Range.bottom);
279 top = new StoreFile(this.fs, topPath, true, conf,
280 StoreFile.BloomType.NONE, false).createReader();
281 bottom = new StoreFile(this.fs, bottomPath, true, conf,
282 StoreFile.BloomType.NONE, false).createReader();
283 first = true;
284 bottomScanner = bottom.getScanner(false, false);
285 while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
286 bottomScanner.next()) {
287 key = bottomScanner.getKey();
288 if (first) {
289 first = false;
290 keyKV = KeyValue.createKeyValueFromKey(key);
291 LOG.info("First bottom when key > top: " + keyKV);
292 tmp = Bytes.toString(keyKV.getRow());
293 for (int i = 0; i < tmp.length(); i++) {
294 assertTrue(tmp.charAt(i) == 'a');
295 }
296 }
297 }
298 keyKV = KeyValue.createKeyValueFromKey(key);
299 LOG.info("Last bottom when key > top: " + keyKV);
300 for (int i = 0; i < tmp.length(); i++) {
301 assertTrue(Bytes.toString(keyKV.getRow()).charAt(i) == 'z');
302 }
303 count = 0;
304 topScanner = top.getScanner(false, false);
305 while ((!topScanner.isSeeked() && topScanner.seekTo()) ||
306 (topScanner.isSeeked() && topScanner.next())) {
307 count++;
308 }
309
310 assertTrue(count == 0);
311 } finally {
312 if (top != null) {
313 top.close();
314 }
315 if (bottom != null) {
316 bottom.close();
317 }
318 fs.delete(f.getPath(), true);
319 }
320 }
321
322 private static String ROOT_DIR =
323 HBaseTestingUtility.getTestDir("TestStoreFile").toString();
324 private static String localFormatter = "%010d";
325
326 public void testBloomFilter() throws Exception {
327 FileSystem fs = FileSystem.getLocal(conf);
328 conf.setFloat("io.hfile.bloom.error.rate", (float)0.01);
329 conf.setBoolean("io.hfile.bloom.enabled", true);
330
331
332 Path f = new Path(ROOT_DIR, getName());
333 StoreFile.Writer writer = new StoreFile.Writer(fs, f,
334 StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
335 conf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
336
337 long now = System.currentTimeMillis();
338 for (int i = 0; i < 2000; i += 2) {
339 String row = String.format(localFormatter, i);
340 KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(),
341 "col".getBytes(), now, "value".getBytes());
342 writer.append(kv);
343 }
344 writer.close();
345
346 StoreFile.Reader reader = new StoreFile.Reader(fs, f, null, false);
347 reader.loadFileInfo();
348 reader.loadBloomfilter();
349 StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
350
351
352 int falsePos = 0;
353 int falseNeg = 0;
354 for (int i = 0; i < 2000; i++) {
355 String row = String.format(localFormatter, i);
356 TreeSet<byte[]> columns = new TreeSet<byte[]>();
357 columns.add("family:col".getBytes());
358
359 Scan scan = new Scan(row.getBytes(),row.getBytes());
360 scan.addColumn("family".getBytes(), "family:col".getBytes());
361 boolean exists = scanner.shouldSeek(scan, columns);
362 if (i % 2 == 0) {
363 if (!exists) falseNeg++;
364 } else {
365 if (exists) falsePos++;
366 }
367 }
368 reader.close();
369 fs.delete(f, true);
370 System.out.println("False negatives: " + falseNeg);
371 assertEquals(0, falseNeg);
372 System.out.println("False positives: " + falsePos);
373 assertTrue(falsePos < 2);
374 }
375
376 public void testBloomTypes() throws Exception {
377 float err = (float) 0.01;
378 FileSystem fs = FileSystem.getLocal(conf);
379 conf.setFloat("io.hfile.bloom.error.rate", err);
380 conf.setBoolean("io.hfile.bloom.enabled", true);
381
382 int rowCount = 50;
383 int colCount = 10;
384 int versions = 2;
385
386
387 StoreFile.BloomType[] bt =
388 {StoreFile.BloomType.ROWCOL, StoreFile.BloomType.ROW};
389 int[] expKeys = {rowCount*colCount, rowCount};
390
391
392
393
394 float[] expErr = {2*rowCount*colCount*err, 2*rowCount*2*colCount*err};
395
396 for (int x : new int[]{0,1}) {
397
398 Path f = new Path(ROOT_DIR, getName());
399 StoreFile.Writer writer = new StoreFile.Writer(fs, f,
400 StoreFile.DEFAULT_BLOCKSIZE_SMALL,
401 HFile.DEFAULT_COMPRESSION_ALGORITHM,
402 conf, KeyValue.COMPARATOR, bt[x], expKeys[x]);
403
404 long now = System.currentTimeMillis();
405 for (int i = 0; i < rowCount*2; i += 2) {
406 for (int j = 0; j < colCount*2; j += 2) {
407 String row = String.format(localFormatter, i);
408 String col = String.format(localFormatter, j);
409 for (int k= 0; k < versions; ++k) {
410 KeyValue kv = new KeyValue(row.getBytes(),
411 "family".getBytes(), ("col" + col).getBytes(),
412 now-k, Bytes.toBytes((long)-1));
413 writer.append(kv);
414 }
415 }
416 }
417 writer.close();
418
419 StoreFile.Reader reader = new StoreFile.Reader(fs, f, null, false);
420 reader.loadFileInfo();
421 reader.loadBloomfilter();
422 StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
423 assertEquals(expKeys[x], reader.bloomFilter.getKeyCount());
424
425
426 int falsePos = 0;
427 int falseNeg = 0;
428 for (int i = 0; i < rowCount*2; ++i) {
429 for (int j = 0; j < colCount*2; ++j) {
430 String row = String.format(localFormatter, i);
431 String col = String.format(localFormatter, j);
432 TreeSet<byte[]> columns = new TreeSet<byte[]>();
433 columns.add(("col" + col).getBytes());
434
435 Scan scan = new Scan(row.getBytes(),row.getBytes());
436 scan.addColumn("family".getBytes(), ("col"+col).getBytes());
437 boolean exists = scanner.shouldSeek(scan, columns);
438 boolean shouldRowExist = i % 2 == 0;
439 boolean shouldColExist = j % 2 == 0;
440 shouldColExist = shouldColExist || bt[x] == StoreFile.BloomType.ROW;
441 if (shouldRowExist && shouldColExist) {
442 if (!exists) falseNeg++;
443 } else {
444 if (exists) falsePos++;
445 }
446 }
447 }
448 reader.close();
449 fs.delete(f, true);
450 System.out.println(bt[x].toString());
451 System.out.println(" False negatives: " + falseNeg);
452 System.out.println(" False positives: " + falsePos);
453 assertEquals(0, falseNeg);
454 assertTrue(falsePos < 2*expErr[x]);
455 }
456 }
457
458 public void testFlushTimeComparator() {
459 assertOrdering(StoreFile.Comparators.FLUSH_TIME,
460 mockStoreFile(true, 1000, -1, "/foo/123"),
461 mockStoreFile(true, 1000, -1, "/foo/126"),
462 mockStoreFile(true, 2000, -1, "/foo/126"),
463 mockStoreFile(false, -1, 1, "/foo/1"),
464 mockStoreFile(false, -1, 3, "/foo/2"),
465 mockStoreFile(false, -1, 5, "/foo/2"),
466 mockStoreFile(false, -1, 5, "/foo/3"));
467 }
468
469
470
471
472
473 private void assertOrdering(Comparator<StoreFile> comparator, StoreFile ... sfs) {
474 ArrayList<StoreFile> sorted = Lists.newArrayList(sfs);
475 Collections.shuffle(sorted);
476 Collections.sort(sorted, comparator);
477 LOG.debug("sfs: " + Joiner.on(",").join(sfs));
478 LOG.debug("sorted: " + Joiner.on(",").join(sorted));
479 assertTrue(Iterables.elementsEqual(Arrays.asList(sfs), sorted));
480 }
481
482
483
484
485 private StoreFile mockStoreFile(boolean bulkLoad, long bulkTimestamp,
486 long seqId, String path) {
487 StoreFile mock = Mockito.mock(StoreFile.class);
488 Mockito.doReturn(bulkLoad).when(mock).isBulkLoadResult();
489 Mockito.doReturn(bulkTimestamp).when(mock).getBulkLoadTimestamp();
490 if (bulkLoad) {
491
492 Mockito.doThrow(new IllegalAccessError("bulk load"))
493 .when(mock).getMaxSequenceId();
494 } else {
495 Mockito.doReturn(seqId).when(mock).getMaxSequenceId();
496 }
497 Mockito.doReturn(new Path(path)).when(mock).getPath();
498 String name = "mock storefile, bulkLoad=" + bulkLoad +
499 " bulkTimestamp=" + bulkTimestamp +
500 " seqId=" + seqId +
501 " path=" + path;
502 Mockito.doReturn(name).when(mock).toString();
503 return mock;
504 }
505
506
507
508
509
510
511
512
513
514 List<KeyValue> getKeyValueSet(long[] timestamps, int numRows,
515 byte[] qualifier, byte[] family) {
516 List<KeyValue> kvList = new ArrayList<KeyValue>();
517 for (int i=1;i<=numRows;i++) {
518 byte[] b = Bytes.toBytes(i) ;
519 LOG.info(Bytes.toString(b));
520 LOG.info(Bytes.toString(b));
521 for (long timestamp: timestamps)
522 {
523 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
524 }
525 }
526 return kvList;
527 }
528
529
530
531
532
533 public void testMultipleTimestamps() throws IOException {
534 byte[] family = Bytes.toBytes("familyname");
535 byte[] qualifier = Bytes.toBytes("qualifier");
536 int numRows = 10;
537 long[] timestamps = new long[] {20,10,5,1};
538 Scan scan = new Scan();
539
540 Path storedir = new Path(new Path(this.testDir, "regionname"),
541 "familyname");
542 Path dir = new Path(storedir, "1234567890");
543 StoreFile.Writer writer = StoreFile.createWriter(this.fs, dir, 8 * 1024);
544
545 List<KeyValue> kvList = getKeyValueSet(timestamps,numRows,
546 family, qualifier);
547
548 for (KeyValue kv : kvList) {
549 writer.append(kv);
550 }
551 writer.appendMetadata(0, false);
552 writer.close();
553
554 StoreFile hsf = new StoreFile(this.fs, writer.getPath(), true, conf,
555 StoreFile.BloomType.NONE, false);
556 StoreFile.Reader reader = hsf.createReader();
557 StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
558 TreeSet<byte[]> columns = new TreeSet<byte[]>();
559 columns.add(qualifier);
560
561 scan.setTimeRange(20, 100);
562 assertTrue(scanner.shouldSeek(scan, columns));
563
564 scan.setTimeRange(1, 2);
565 assertTrue(scanner.shouldSeek(scan, columns));
566
567 scan.setTimeRange(8, 10);
568 assertTrue(scanner.shouldSeek(scan, columns));
569
570 scan.setTimeRange(7, 50);
571 assertTrue(scanner.shouldSeek(scan, columns));
572
573
574
575
576
577 }
578 }