1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Collections;
27 import java.util.Comparator;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.TreeSet;
31 import java.util.regex.Pattern;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileStatus;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.HBaseTestCase;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.KeyValue;
43 import org.apache.hadoop.hbase.SmallTests;
44 import org.apache.hadoop.hbase.client.Scan;
45 import org.apache.hadoop.hbase.io.HFileLink;
46 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
47 import org.apache.hadoop.hbase.io.Reference;
48 import org.apache.hadoop.hbase.io.Reference.Range;
49 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
50 import org.apache.hadoop.hbase.io.hfile.BlockCache;
51 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
52 import org.apache.hadoop.hbase.io.hfile.CacheStats;
53 import org.apache.hadoop.hbase.io.hfile.HFile;
54 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
55 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
56 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
57 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
58 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
59 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
60 import org.apache.hadoop.hbase.util.BloomFilterFactory;
61 import org.apache.hadoop.hbase.util.Bytes;
62 import org.apache.hadoop.hbase.util.ChecksumType;
63 import org.apache.hadoop.hbase.util.FSUtils;
64 import org.junit.experimental.categories.Category;
65 import org.mockito.Mockito;
66
67 import com.google.common.base.Joiner;
68 import com.google.common.collect.Iterables;
69 import com.google.common.collect.Lists;
70
71
72
73
74 @Category(SmallTests.class)
75 public class TestStoreFile extends HBaseTestCase {
76 static final Log LOG = LogFactory.getLog(TestStoreFile.class);
77 private CacheConfig cacheConf = new CacheConfig(conf);
78 private String ROOT_DIR;
79 private Map<String, Long> startingMetrics;
80
81 private static final ChecksumType CKTYPE = ChecksumType.CRC32;
82 private static final int CKBYTES = 512;
83
84 @Override
85 public void setUp() throws Exception {
86 super.setUp();
87 startingMetrics = SchemaMetrics.getMetricsSnapshot();
88 ROOT_DIR = new Path(this.testDir, "TestStoreFile").toString();
89 }
90
91 @Override
92 public void tearDown() throws Exception {
93 super.tearDown();
94 SchemaMetrics.validateMetricChanges(startingMetrics);
95 }
96
97
98
99
100
101
102 public void testBasicHalfMapFile() throws Exception {
103
104 Path outputDir = new Path(new Path(this.testDir, "7e0102"),
105 "familyname");
106 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
107 this.fs, 2 * 1024)
108 .withOutputDir(outputDir)
109 .build();
110 writeStoreFile(writer);
111 checkHalfHFile(new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
112 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE));
113 }
114
115 private void writeStoreFile(final StoreFile.Writer writer) throws IOException {
116 writeStoreFile(writer, Bytes.toBytes(getName()), Bytes.toBytes(getName()));
117 }
118
119
120 byte[] SPLITKEY = new byte[] { (LAST_CHAR + FIRST_CHAR)/2, FIRST_CHAR};
121
122
123
124
125
126
127
128 public static void writeStoreFile(final StoreFile.Writer writer, byte[] fam, byte[] qualifier)
129 throws IOException {
130 long now = System.currentTimeMillis();
131 try {
132 for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {
133 for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) {
134 byte[] b = new byte[] { (byte) d, (byte) e };
135 writer.append(new KeyValue(b, fam, qualifier, now, b));
136 }
137 }
138 } finally {
139 writer.close();
140 }
141 }
142
143
144
145
146
147
148 public void testReference()
149 throws IOException {
150
151 Path storedir = new Path(new Path(this.testDir, "7e0102"), "familyname");
152
153 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
154 this.fs, 8 * 1024)
155 .withOutputDir(storedir)
156 .build();
157 writeStoreFile(writer);
158 StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
159 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
160 StoreFile.Reader reader = hsf.createReader();
161
162
163
164 KeyValue kv = KeyValue.createKeyValueFromKey(reader.midkey());
165 byte [] midRow = kv.getRow();
166 kv = KeyValue.createKeyValueFromKey(reader.getLastKey());
167 byte [] finalRow = kv.getRow();
168
169 Path refPath = StoreFile.split(fs, storedir, hsf, midRow, Range.top);
170 StoreFile refHsf = new StoreFile(this.fs, refPath, conf, cacheConf,
171 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
172
173
174 HFileScanner s = refHsf.createReader().getScanner(false, false);
175 for(boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {
176 ByteBuffer bb = s.getKey();
177 kv = KeyValue.createKeyValueFromKey(bb);
178 if (first) {
179 assertTrue(Bytes.equals(kv.getRow(), midRow));
180 first = false;
181 }
182 }
183 assertTrue(Bytes.equals(kv.getRow(), finalRow));
184 }
185
186 public void testHFileLink() throws IOException {
187 final String columnFamily = "f";
188
189 Configuration testConf = new Configuration(this.conf);
190 FSUtils.setRootDir(testConf, this.testDir);
191
192 HRegionInfo hri = new HRegionInfo(Bytes.toBytes("table-link"));
193 Path storedir = new Path(new Path(this.testDir,
194 new Path(hri.getTableNameAsString(), hri.getEncodedName())), columnFamily);
195
196
197 StoreFile.Writer writer = new StoreFile.WriterBuilder(testConf, cacheConf,
198 this.fs, 8 * 1024)
199 .withOutputDir(storedir)
200 .build();
201 Path storeFilePath = writer.getPath();
202 writeStoreFile(writer);
203 writer.close();
204
205 Path dstPath = new Path(this.testDir, new Path("test-region", columnFamily));
206 HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
207 Path linkFilePath = new Path(dstPath,
208 HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
209
210
211 StoreFile hsf = new StoreFile(this.fs, linkFilePath, testConf, cacheConf,
212 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
213 assertTrue(hsf.isLink());
214
215
216 int count = 1;
217 HFileScanner s = hsf.createReader().getScanner(false, false);
218 s.seekTo();
219 while (s.next()) {
220 count++;
221 }
222 assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
223 }
224
225
226
227
228 public void testStoreFileNames() {
229 String[] legalHFileLink = { "MyTable_02=abc012-def345", "MyTable_02.300=abc012-def345",
230 "MyTable_02-400=abc012-def345", "MyTable_02-400.200=abc012-def345",
231 "MyTable_02=abc012-def345_SeqId_1_", "MyTable_02=abc012-def345_SeqId_20_" };
232 for (String name: legalHFileLink) {
233 assertTrue("should be a valid link: " + name, HFileLink.isHFileLink(name));
234 assertTrue("should be a valid StoreFile" + name, StoreFile.validateStoreFileName(name));
235 assertFalse("should not be a valid reference: " + name, StoreFile.isReference(name));
236
237 String refName = name + ".6789";
238 assertTrue("should be a valid link reference: " + refName, StoreFile.isReference(refName));
239 assertTrue("should be a valid StoreFile" + refName, StoreFile.validateStoreFileName(refName));
240 }
241
242 String[] illegalHFileLink = { ".MyTable_02=abc012-def345", "-MyTable_02.300=abc012-def345",
243 "MyTable_02-400=abc0_12-def345", "MyTable_02-400.200=abc012-def345...." };
244 for (String name: illegalHFileLink) {
245 assertFalse("should not be a valid link: " + name, HFileLink.isHFileLink(name));
246 }
247 }
248
249
250
251
252
253 public void testReferenceToHFileLink() throws IOException {
254 final String columnFamily = "f";
255
256 Path rootDir = FSUtils.getRootDir(conf);
257
258 String tablename = "_original-evil-name";
259 HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tablename));
260
261 Path storedir = new Path(new Path(rootDir,
262 new Path(hri.getTableNameAsString(), hri.getEncodedName())), columnFamily);
263
264
265 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
266 this.fs, 8 * 1024)
267 .withOutputDir(storedir)
268 .build();
269 Path storeFilePath = writer.getPath();
270 writeStoreFile(writer);
271 writer.close();
272
273
274 String target = "clone";
275 Path dstPath = new Path(rootDir, new Path(new Path(target, "7e0102"), columnFamily));
276 HFileLink.create(conf, this.fs, dstPath, hri, storeFilePath.getName());
277 Path linkFilePath = new Path(dstPath,
278 HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
279
280
281
282
283 Path splitDirA = new Path(new Path(rootDir,
284 new Path(target, "571A")), columnFamily);
285 Path splitDirB = new Path(new Path(rootDir,
286 new Path(target, "571B")), columnFamily);
287 StoreFile f = new StoreFile(fs, linkFilePath, conf, cacheConf, BloomType.NONE,
288 NoOpDataBlockEncoder.INSTANCE);
289 byte[] splitRow = SPLITKEY;
290 Path pathA = StoreFile.split(fs, splitDirA, f, splitRow, Range.top);
291 Path pathB = StoreFile.split(fs, splitDirB, f, splitRow, Range.bottom);
292
293
294 FSUtils.logFileSystemState(fs, rootDir, LOG);
295
296
297
298
299
300 StoreFile hsfA = new StoreFile(this.fs, pathA, conf, cacheConf,
301 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
302
303
304 int count = 1;
305 HFileScanner s = hsfA.createReader().getScanner(false, false);
306 s.seekTo();
307 while (s.next()) {
308 count++;
309 }
310 assertTrue(count > 0);
311
312
313 StoreFile hsfB = new StoreFile(this.fs, pathB, conf, cacheConf,
314 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
315
316
317 HFileScanner sB = hsfB.createReader().getScanner(false, false);
318 sB.seekTo();
319
320
321 count++;
322 while (sB.next()) {
323 count++;
324 }
325
326
327 assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
328 }
329
330 private void checkHalfHFile(final StoreFile f)
331 throws IOException {
332 byte [] midkey = f.createReader().midkey();
333 KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
334 byte [] midRow = midKV.getRow();
335
336 Path topDir = Store.getStoreHomedir(this.testDir, "1",
337 Bytes.toBytes(f.getPath().getParent().getName()));
338 if (this.fs.exists(topDir)) {
339 this.fs.delete(topDir, true);
340 }
341 Path topPath = StoreFile.split(this.fs, topDir, f, midRow, Range.top);
342
343 Path bottomDir = Store.getStoreHomedir(this.testDir, "2",
344 Bytes.toBytes(f.getPath().getParent().getName()));
345 if (this.fs.exists(bottomDir)) {
346 this.fs.delete(bottomDir, true);
347 }
348 Path bottomPath = StoreFile.split(this.fs, bottomDir,
349 f, midRow, Range.bottom);
350
351 StoreFile.Reader top =
352 new StoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE,
353 NoOpDataBlockEncoder.INSTANCE).createReader();
354 StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath,
355 conf, cacheConf, BloomType.NONE,
356 NoOpDataBlockEncoder.INSTANCE).createReader();
357 ByteBuffer previous = null;
358 LOG.info("Midkey: " + midKV.toString());
359 ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midkey);
360 try {
361
362
363
364
365 boolean first = true;
366 ByteBuffer key = null;
367 HFileScanner topScanner = top.getScanner(false, false);
368 while ((!topScanner.isSeeked() && topScanner.seekTo()) ||
369 (topScanner.isSeeked() && topScanner.next())) {
370 key = topScanner.getKey();
371
372 if (topScanner.getReader().getComparator().compare(key.array(),
373 key.arrayOffset(), key.limit(), midkey, 0, midkey.length) < 0) {
374 fail("key=" + Bytes.toStringBinary(key) + " < midkey=" +
375 Bytes.toStringBinary(midkey));
376 }
377 if (first) {
378 first = false;
379 LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key)));
380 }
381 }
382 LOG.info("Last in top: " + Bytes.toString(Bytes.toBytes(key)));
383
384 first = true;
385 HFileScanner bottomScanner = bottom.getScanner(false, false);
386 while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
387 bottomScanner.next()) {
388 previous = bottomScanner.getKey();
389 key = bottomScanner.getKey();
390 if (first) {
391 first = false;
392 LOG.info("First in bottom: " +
393 Bytes.toString(Bytes.toBytes(previous)));
394 }
395 assertTrue(key.compareTo(bbMidkeyBytes) < 0);
396 }
397 if (previous != null) {
398 LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
399 }
400
401 this.fs.delete(topPath, false);
402 this.fs.delete(bottomPath, false);
403
404
405
406
407 byte [] badmidkey = Bytes.toBytes(" .");
408 topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top);
409 bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
410 Range.bottom);
411
412 assertNull(bottomPath);
413
414 top = new StoreFile(this.fs, topPath, conf, cacheConf,
415 StoreFile.BloomType.NONE,
416 NoOpDataBlockEncoder.INSTANCE).createReader();
417
418 first = true;
419 topScanner = top.getScanner(false, false);
420 while ((!topScanner.isSeeked() && topScanner.seekTo()) ||
421 topScanner.next()) {
422 key = topScanner.getKey();
423 assertTrue(topScanner.getReader().getComparator().compare(key.array(),
424 key.arrayOffset(), key.limit(), badmidkey, 0, badmidkey.length) >= 0);
425 if (first) {
426 first = false;
427 KeyValue keyKV = KeyValue.createKeyValueFromKey(key);
428 LOG.info("First top when key < bottom: " + keyKV);
429 String tmp = Bytes.toString(keyKV.getRow());
430 for (int i = 0; i < tmp.length(); i++) {
431 assertTrue(tmp.charAt(i) == 'a');
432 }
433 }
434 }
435 KeyValue keyKV = KeyValue.createKeyValueFromKey(key);
436 LOG.info("Last top when key < bottom: " + keyKV);
437 String tmp = Bytes.toString(keyKV.getRow());
438 for (int i = 0; i < tmp.length(); i++) {
439 assertTrue(tmp.charAt(i) == 'z');
440 }
441
442 this.fs.delete(topPath, false);
443
444
445 badmidkey = Bytes.toBytes("|||");
446 topPath = StoreFile.split(this.fs, topDir, f, badmidkey, Range.top);
447 bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
448 Range.bottom);
449
450 assertNull(topPath);
451
452 bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
453 StoreFile.BloomType.NONE,
454 NoOpDataBlockEncoder.INSTANCE).createReader();
455 first = true;
456 bottomScanner = bottom.getScanner(false, false);
457 while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
458 bottomScanner.next()) {
459 key = bottomScanner.getKey();
460 if (first) {
461 first = false;
462 keyKV = KeyValue.createKeyValueFromKey(key);
463 LOG.info("First bottom when key > top: " + keyKV);
464 tmp = Bytes.toString(keyKV.getRow());
465 for (int i = 0; i < tmp.length(); i++) {
466 assertTrue(tmp.charAt(i) == 'a');
467 }
468 }
469 }
470 keyKV = KeyValue.createKeyValueFromKey(key);
471 LOG.info("Last bottom when key > top: " + keyKV);
472 for (int i = 0; i < tmp.length(); i++) {
473 assertTrue(Bytes.toString(keyKV.getRow()).charAt(i) == 'z');
474 }
475 } finally {
476 if (top != null) {
477 top.close(true);
478 }
479 if (bottom != null) {
480 bottom.close(true);
481 }
482 fs.delete(f.getPath(), true);
483 }
484 }
485
486 private static final String localFormatter = "%010d";
487
488 private void bloomWriteRead(StoreFile.Writer writer, FileSystem fs)
489 throws Exception {
490 float err = conf.getFloat(
491 BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, 0);
492 Path f = writer.getPath();
493 long now = System.currentTimeMillis();
494 for (int i = 0; i < 2000; i += 2) {
495 String row = String.format(localFormatter, i);
496 KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(),
497 "col".getBytes(), now, "value".getBytes());
498 writer.append(kv);
499 }
500 writer.close();
501
502 StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
503 DataBlockEncoding.NONE);
504 reader.loadFileInfo();
505 reader.loadBloomfilter();
506 StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
507
508
509 int falsePos = 0;
510 int falseNeg = 0;
511 for (int i = 0; i < 2000; i++) {
512 String row = String.format(localFormatter, i);
513 TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
514 columns.add("family:col".getBytes());
515
516 Scan scan = new Scan(row.getBytes(),row.getBytes());
517 scan.addColumn("family".getBytes(), "family:col".getBytes());
518 boolean exists = scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE);
519 if (i % 2 == 0) {
520 if (!exists) falseNeg++;
521 } else {
522 if (exists) falsePos++;
523 }
524 }
525 reader.close(true);
526 fs.delete(f, true);
527 assertEquals("False negatives: " + falseNeg, 0, falseNeg);
528 int maxFalsePos = (int) (2 * 2000 * err);
529 assertTrue("Too many false positives: " + falsePos + " (err=" + err
530 + ", expected no more than " + maxFalsePos + ")",
531 falsePos <= maxFalsePos);
532 }
533
534 public void testBloomFilter() throws Exception {
535 FileSystem fs = FileSystem.getLocal(conf);
536 conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE,
537 (float) 0.01);
538 conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
539
540
541 Path f = new Path(ROOT_DIR, getName());
542 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
543 StoreFile.DEFAULT_BLOCKSIZE_SMALL)
544 .withFilePath(f)
545 .withBloomType(StoreFile.BloomType.ROW)
546 .withMaxKeyCount(2000)
547 .withChecksumType(CKTYPE)
548 .withBytesPerChecksum(CKBYTES)
549 .build();
550 bloomWriteRead(writer, fs);
551 }
552
553 public void testDeleteFamilyBloomFilter() throws Exception {
554 FileSystem fs = FileSystem.getLocal(conf);
555 conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE,
556 (float) 0.01);
557 conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
558 float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE,
559 0);
560
561
562 Path f = new Path(ROOT_DIR, getName());
563
564 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
565 fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL)
566 .withFilePath(f)
567 .withMaxKeyCount(2000)
568 .withChecksumType(CKTYPE)
569 .withBytesPerChecksum(CKBYTES)
570 .build();
571
572
573 long now = System.currentTimeMillis();
574 for (int i = 0; i < 2000; i += 2) {
575 String row = String.format(localFormatter, i);
576 KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(),
577 "col".getBytes(), now, KeyValue.Type.DeleteFamily, "value".getBytes());
578 writer.append(kv);
579 }
580 writer.close();
581
582 StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
583 DataBlockEncoding.NONE);
584 reader.loadFileInfo();
585 reader.loadBloomfilter();
586
587
588 int falsePos = 0;
589 int falseNeg = 0;
590 for (int i = 0; i < 2000; i++) {
591 String row = String.format(localFormatter, i);
592 byte[] rowKey = Bytes.toBytes(row);
593 boolean exists = reader.passesDeleteFamilyBloomFilter(rowKey, 0,
594 rowKey.length);
595 if (i % 2 == 0) {
596 if (!exists)
597 falseNeg++;
598 } else {
599 if (exists)
600 falsePos++;
601 }
602 }
603 assertEquals(1000, reader.getDeleteFamilyCnt());
604 reader.close(true);
605 fs.delete(f, true);
606 assertEquals("False negatives: " + falseNeg, 0, falseNeg);
607 int maxFalsePos = (int) (2 * 2000 * err);
608 assertTrue("Too many false positives: " + falsePos + " (err=" + err
609 + ", expected no more than " + maxFalsePos, falsePos <= maxFalsePos);
610 }
611
612
613
614
615 public void testReseek() throws Exception {
616
617 Path f = new Path(ROOT_DIR, getName());
618
619
620 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
621 this.fs, 8 * 1024)
622 .withFilePath(f)
623 .build();
624
625 writeStoreFile(writer);
626 writer.close();
627
628 StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf, DataBlockEncoding.NONE);
629
630
631
632 KeyValue k = KeyValue.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY);
633 StoreFileScanner s = reader.getStoreFileScanner(false, false);
634 s.reseek(k);
635
636 assertNotNull("Intial reseek should position at the beginning of the file", s.peek());
637 }
638
639 public void testBloomTypes() throws Exception {
640 float err = (float) 0.01;
641 FileSystem fs = FileSystem.getLocal(conf);
642 conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err);
643 conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
644
645 int rowCount = 50;
646 int colCount = 10;
647 int versions = 2;
648
649
650 StoreFile.BloomType[] bt =
651 {StoreFile.BloomType.ROWCOL, StoreFile.BloomType.ROW};
652 int[] expKeys = {rowCount*colCount, rowCount};
653
654
655
656
657 float[] expErr = {2*rowCount*colCount*err, 2*rowCount*2*colCount*err};
658
659 for (int x : new int[]{0,1}) {
660
661 Path f = new Path(ROOT_DIR, getName() + x);
662 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
663 fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL)
664 .withFilePath(f)
665 .withBloomType(bt[x])
666 .withMaxKeyCount(expKeys[x])
667 .withChecksumType(CKTYPE)
668 .withBytesPerChecksum(CKBYTES)
669 .build();
670
671 long now = System.currentTimeMillis();
672 for (int i = 0; i < rowCount*2; i += 2) {
673 for (int j = 0; j < colCount*2; j += 2) {
674 String row = String.format(localFormatter, i);
675 String col = String.format(localFormatter, j);
676 for (int k= 0; k < versions; ++k) {
677 KeyValue kv = new KeyValue(row.getBytes(),
678 "family".getBytes(), ("col" + col).getBytes(),
679 now-k, Bytes.toBytes((long)-1));
680 writer.append(kv);
681 }
682 }
683 }
684 writer.close();
685
686 StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
687 DataBlockEncoding.NONE);
688 reader.loadFileInfo();
689 reader.loadBloomfilter();
690 StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
691 assertEquals(expKeys[x], reader.generalBloomFilter.getKeyCount());
692
693
694 int falsePos = 0;
695 int falseNeg = 0;
696 for (int i = 0; i < rowCount*2; ++i) {
697 for (int j = 0; j < colCount*2; ++j) {
698 String row = String.format(localFormatter, i);
699 String col = String.format(localFormatter, j);
700 TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
701 columns.add(("col" + col).getBytes());
702
703 Scan scan = new Scan(row.getBytes(),row.getBytes());
704 scan.addColumn("family".getBytes(), ("col"+col).getBytes());
705 boolean exists =
706 scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE);
707 boolean shouldRowExist = i % 2 == 0;
708 boolean shouldColExist = j % 2 == 0;
709 shouldColExist = shouldColExist || bt[x] == StoreFile.BloomType.ROW;
710 if (shouldRowExist && shouldColExist) {
711 if (!exists) falseNeg++;
712 } else {
713 if (exists) falsePos++;
714 }
715 }
716 }
717 reader.close(true);
718 fs.delete(f, true);
719 System.out.println(bt[x].toString());
720 System.out.println(" False negatives: " + falseNeg);
721 System.out.println(" False positives: " + falsePos);
722 assertEquals(0, falseNeg);
723 assertTrue(falsePos < 2*expErr[x]);
724 }
725 }
726
727 public void testBloomEdgeCases() throws Exception {
728 float err = (float)0.005;
729 FileSystem fs = FileSystem.getLocal(conf);
730 Path f = new Path(ROOT_DIR, getName());
731 conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err);
732 conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
733 conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_MAX_KEYS, 1000);
734
735
736 conf.setInt(HFile.FORMAT_VERSION_KEY, 1);
737
738
739 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
740 StoreFile.DEFAULT_BLOCKSIZE_SMALL)
741 .withFilePath(f)
742 .withBloomType(StoreFile.BloomType.ROW)
743 .withMaxKeyCount(2000)
744 .withChecksumType(CKTYPE)
745 .withBytesPerChecksum(CKBYTES)
746 .build();
747 assertFalse(writer.hasGeneralBloom());
748 writer.close();
749 fs.delete(f, true);
750
751 conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_MAX_KEYS,
752 Integer.MAX_VALUE);
753
754
755
756
757 writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
758 StoreFile.DEFAULT_BLOCKSIZE_SMALL)
759 .withFilePath(f)
760 .withBloomType(StoreFile.BloomType.ROW)
761 .withMaxKeyCount(27244696)
762 .build();
763 assertTrue(writer.hasGeneralBloom());
764 bloomWriteRead(writer, fs);
765
766
767
768 writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
769 StoreFile.DEFAULT_BLOCKSIZE_SMALL)
770 .withFilePath(f)
771 .withBloomType(StoreFile.BloomType.ROW)
772 .withMaxKeyCount(Integer.MAX_VALUE)
773 .withChecksumType(CKTYPE)
774 .withBytesPerChecksum(CKBYTES)
775 .build();
776 assertFalse(writer.hasGeneralBloom());
777 writer.close();
778 fs.delete(f, true);
779 }
780
781 public void testFlushTimeComparator() {
782 assertOrdering(StoreFile.Comparators.FLUSH_TIME,
783 mockStoreFile(true, 1000, -1, "/foo/123"),
784 mockStoreFile(true, 1000, -1, "/foo/126"),
785 mockStoreFile(true, 2000, -1, "/foo/126"),
786 mockStoreFile(false, -1, 1, "/foo/1"),
787 mockStoreFile(false, -1, 3, "/foo/2"),
788 mockStoreFile(false, -1, 5, "/foo/2"),
789 mockStoreFile(false, -1, 5, "/foo/3"));
790 }
791
792
793
794
795
796 private void assertOrdering(Comparator<StoreFile> comparator, StoreFile ... sfs) {
797 ArrayList<StoreFile> sorted = Lists.newArrayList(sfs);
798 Collections.shuffle(sorted);
799 Collections.sort(sorted, comparator);
800 LOG.debug("sfs: " + Joiner.on(",").join(sfs));
801 LOG.debug("sorted: " + Joiner.on(",").join(sorted));
802 assertTrue(Iterables.elementsEqual(Arrays.asList(sfs), sorted));
803 }
804
805
806
807
808 private StoreFile mockStoreFile(boolean bulkLoad, long bulkTimestamp,
809 long seqId, String path) {
810 StoreFile mock = Mockito.mock(StoreFile.class);
811 Mockito.doReturn(bulkLoad).when(mock).isBulkLoadResult();
812 Mockito.doReturn(bulkTimestamp).when(mock).getBulkLoadTimestamp();
813 if (bulkLoad) {
814
815 Mockito.doThrow(new IllegalAccessError("bulk load"))
816 .when(mock).getMaxSequenceId();
817 } else {
818 Mockito.doReturn(seqId).when(mock).getMaxSequenceId();
819 }
820 Mockito.doReturn(new Path(path)).when(mock).getPath();
821 String name = "mock storefile, bulkLoad=" + bulkLoad +
822 " bulkTimestamp=" + bulkTimestamp +
823 " seqId=" + seqId +
824 " path=" + path;
825 Mockito.doReturn(name).when(mock).toString();
826 return mock;
827 }
828
829
830
831
832
833
834
835
836
837 List<KeyValue> getKeyValueSet(long[] timestamps, int numRows,
838 byte[] qualifier, byte[] family) {
839 List<KeyValue> kvList = new ArrayList<KeyValue>();
840 for (int i=1;i<=numRows;i++) {
841 byte[] b = Bytes.toBytes(i) ;
842 LOG.info(Bytes.toString(b));
843 LOG.info(Bytes.toString(b));
844 for (long timestamp: timestamps)
845 {
846 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
847 }
848 }
849 return kvList;
850 }
851
852
853
854
855
856 public void testMultipleTimestamps() throws IOException {
857 byte[] family = Bytes.toBytes("familyname");
858 byte[] qualifier = Bytes.toBytes("qualifier");
859 int numRows = 10;
860 long[] timestamps = new long[] {20,10,5,1};
861 Scan scan = new Scan();
862
863
864 Path storedir = new Path(new Path(this.testDir, "7e0102"), "familyname");
865 Path dir = new Path(storedir, "1234567890");
866 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf,
867 this.fs, 8 * 1024)
868 .withOutputDir(dir)
869 .build();
870
871 List<KeyValue> kvList = getKeyValueSet(timestamps,numRows,
872 family, qualifier);
873
874 for (KeyValue kv : kvList) {
875 writer.append(kv);
876 }
877 writer.appendMetadata(0, false);
878 writer.close();
879
880 StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
881 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
882 StoreFile.Reader reader = hsf.createReader();
883 StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
884 TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
885 columns.add(qualifier);
886
887 scan.setTimeRange(20, 100);
888 assertTrue(scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE));
889
890 scan.setTimeRange(1, 2);
891 assertTrue(scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE));
892
893 scan.setTimeRange(8, 10);
894 assertTrue(scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE));
895
896 scan.setTimeRange(7, 50);
897 assertTrue(scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE));
898
899
900 scan.setTimeRange(27, 50);
901 assertTrue(!scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE));
902 }
903
904 public void testCacheOnWriteEvictOnClose() throws Exception {
905 Configuration conf = this.conf;
906
907
908 Path baseDir = new Path(new Path(this.testDir, "7e0102"),"twoCOWEOC");
909
910
911 BlockCache bc = new CacheConfig(conf).getBlockCache();
912 assertNotNull(bc);
913 CacheStats cs = bc.getStats();
914 long startHit = cs.getHitCount();
915 long startMiss = cs.getMissCount();
916 long startEvicted = cs.getEvictedCount();
917
918
919 conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
920 CacheConfig cacheConf = new CacheConfig(conf);
921 Path pathCowOff = new Path(baseDir, "123456789");
922 StoreFile.Writer writer = writeStoreFile(conf, cacheConf, pathCowOff, 3);
923 StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
924 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
925 LOG.debug(hsf.getPath().toString());
926
927
928 StoreFile.Reader reader = hsf.createReader();
929 reader.loadFileInfo();
930 StoreFileScanner scanner = reader.getStoreFileScanner(true, true);
931 scanner.seek(KeyValue.LOWESTKEY);
932 while (scanner.next() != null);
933 assertEquals(startHit, cs.getHitCount());
934 assertEquals(startMiss + 3, cs.getMissCount());
935 assertEquals(startEvicted, cs.getEvictedCount());
936 startMiss += 3;
937 scanner.close();
938 reader.close(cacheConf.shouldEvictOnClose());
939
940
941 conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true);
942 cacheConf = new CacheConfig(conf);
943 Path pathCowOn = new Path(baseDir, "123456788");
944 writer = writeStoreFile(conf, cacheConf, pathCowOn, 3);
945 hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
946 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
947
948
949 reader = hsf.createReader();
950 scanner = reader.getStoreFileScanner(true, true);
951 scanner.seek(KeyValue.LOWESTKEY);
952 while (scanner.next() != null);
953 assertEquals(startHit + 3, cs.getHitCount());
954 assertEquals(startMiss, cs.getMissCount());
955 assertEquals(startEvicted, cs.getEvictedCount());
956 startHit += 3;
957 scanner.close();
958 reader.close(cacheConf.shouldEvictOnClose());
959
960
961 hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf,
962 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
963 StoreFile.Reader readerOne = hsf.createReader();
964 readerOne.loadFileInfo();
965 StoreFileScanner scannerOne = readerOne.getStoreFileScanner(true, true);
966 scannerOne.seek(KeyValue.LOWESTKEY);
967 hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf,
968 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
969 StoreFile.Reader readerTwo = hsf.createReader();
970 readerTwo.loadFileInfo();
971 StoreFileScanner scannerTwo = readerTwo.getStoreFileScanner(true, true);
972 scannerTwo.seek(KeyValue.LOWESTKEY);
973 KeyValue kv1 = null;
974 KeyValue kv2 = null;
975 while ((kv1 = scannerOne.next()) != null) {
976 kv2 = scannerTwo.next();
977 assertTrue(kv1.equals(kv2));
978 assertTrue(Bytes.compareTo(
979 kv1.getBuffer(), kv1.getKeyOffset(), kv1.getKeyLength(),
980 kv2.getBuffer(), kv2.getKeyOffset(), kv2.getKeyLength()) == 0);
981 assertTrue(Bytes.compareTo(
982 kv1.getBuffer(), kv1.getValueOffset(), kv1.getValueLength(),
983 kv2.getBuffer(), kv2.getValueOffset(), kv2.getValueLength()) == 0);
984 }
985 assertNull(scannerTwo.next());
986 assertEquals(startHit + 6, cs.getHitCount());
987 assertEquals(startMiss, cs.getMissCount());
988 assertEquals(startEvicted, cs.getEvictedCount());
989 startHit += 6;
990 scannerOne.close();
991 readerOne.close(cacheConf.shouldEvictOnClose());
992 scannerTwo.close();
993 readerTwo.close(cacheConf.shouldEvictOnClose());
994
995
996 conf.setBoolean("hbase.rs.evictblocksonclose", true);
997 cacheConf = new CacheConfig(conf);
998 hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf,
999 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
1000 reader = hsf.createReader();
1001 reader.close(cacheConf.shouldEvictOnClose());
1002
1003
1004 assertEquals(startHit, cs.getHitCount());
1005 assertEquals(startMiss, cs.getMissCount());
1006 assertEquals(startEvicted + 3, cs.getEvictedCount());
1007 startEvicted += 3;
1008
1009
1010 conf.setBoolean("hbase.rs.evictblocksonclose", false);
1011 cacheConf = new CacheConfig(conf);
1012 hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf,
1013 StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
1014 reader = hsf.createReader();
1015 reader.close(cacheConf.shouldEvictOnClose());
1016
1017
1018 assertEquals(startHit, cs.getHitCount());
1019 assertEquals(startMiss, cs.getMissCount());
1020 assertEquals(startEvicted, cs.getEvictedCount());
1021 }
1022
1023 private StoreFile.Writer writeStoreFile(Configuration conf,
1024 CacheConfig cacheConf, Path path, int numBlocks)
1025 throws IOException {
1026
1027 int numKVs = 5 * numBlocks;
1028 List<KeyValue> kvs = new ArrayList<KeyValue>(numKVs);
1029 byte [] b = Bytes.toBytes("x");
1030 int totalSize = 0;
1031 for (int i=numKVs;i>0;i--) {
1032 KeyValue kv = new KeyValue(b, b, b, i, b);
1033 kvs.add(kv);
1034
1035 totalSize += kv.getLength() + 1;
1036 }
1037 int blockSize = totalSize / numBlocks;
1038 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
1039 blockSize)
1040 .withFilePath(path)
1041 .withMaxKeyCount(2000)
1042 .withChecksumType(CKTYPE)
1043 .withBytesPerChecksum(CKBYTES)
1044 .build();
1045
1046 kvs.remove(kvs.size()-1);
1047 for (KeyValue kv : kvs) {
1048 writer.append(kv);
1049 }
1050 writer.appendMetadata(0, false);
1051 writer.close();
1052 return writer;
1053 }
1054
1055
1056
1057
1058
1059 public void testDataBlockEncodingMetaData() throws IOException {
1060
1061 Path dir = new Path(new Path(this.testDir, "7e0102"), "familyname");
1062 Path path = new Path(dir, "1234567890");
1063
1064 DataBlockEncoding dataBlockEncoderAlgo =
1065 DataBlockEncoding.FAST_DIFF;
1066 HFileDataBlockEncoder dataBlockEncoder =
1067 new HFileDataBlockEncoderImpl(
1068 dataBlockEncoderAlgo,
1069 dataBlockEncoderAlgo);
1070 cacheConf = new CacheConfig(conf);
1071 StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
1072 HFile.DEFAULT_BLOCKSIZE)
1073 .withFilePath(path)
1074 .withDataBlockEncoder(dataBlockEncoder)
1075 .withMaxKeyCount(2000)
1076 .withChecksumType(CKTYPE)
1077 .withBytesPerChecksum(CKBYTES)
1078 .build();
1079 writer.close();
1080
1081 StoreFile storeFile = new StoreFile(fs, writer.getPath(), conf,
1082 cacheConf, BloomType.NONE, dataBlockEncoder);
1083 StoreFile.Reader reader = storeFile.createReader();
1084
1085 Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
1086 byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
1087
1088 assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
1089 }
1090
1091 @org.junit.Rule
1092 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1093 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1094 }
1095