1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.lang.ref.SoftReference;
24 import java.security.PrivilegedExceptionAction;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.NavigableSet;
31 import java.util.concurrent.ConcurrentSkipListSet;
32
33 import junit.framework.TestCase;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FSDataOutputStream;
39 import org.apache.hadoop.fs.FileStatus;
40 import org.apache.hadoop.fs.FileSystem;
41 import org.apache.hadoop.fs.FilterFileSystem;
42 import org.apache.hadoop.fs.LocalFileSystem;
43 import org.apache.hadoop.fs.Path;
44 import org.apache.hadoop.fs.permission.FsPermission;
45 import org.apache.hadoop.hbase.Cell;
46 import org.apache.hadoop.hbase.CellUtil;
47 import org.apache.hadoop.hbase.HBaseConfiguration;
48 import org.apache.hadoop.hbase.HBaseTestingUtility;
49 import org.apache.hadoop.hbase.HColumnDescriptor;
50 import org.apache.hadoop.hbase.HRegionInfo;
51 import org.apache.hadoop.hbase.HTableDescriptor;
52 import org.apache.hadoop.hbase.KeyValue;
53 import org.apache.hadoop.hbase.KeyValue.KVComparator;
54 import org.apache.hadoop.hbase.KeyValueUtil;
55 import org.apache.hadoop.hbase.MediumTests;
56 import org.apache.hadoop.hbase.TableName;
57 import org.apache.hadoop.hbase.client.Get;
58 import org.apache.hadoop.hbase.io.compress.Compression;
59 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
60 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
61 import org.apache.hadoop.hbase.io.hfile.HFile;
62 import org.apache.hadoop.hbase.io.hfile.HFileContext;
63 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
64 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
65 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
66 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
67 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
68 import org.apache.hadoop.hbase.regionserver.wal.HLog;
69 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
70 import org.apache.hadoop.hbase.security.User;
71 import org.apache.hadoop.hbase.util.Bytes;
72 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
74 import org.apache.hadoop.hbase.util.FSUtils;
75 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
76 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
77 import org.apache.hadoop.util.Progressable;
78 import org.junit.experimental.categories.Category;
79 import org.mockito.Mockito;
80
81
82
83
84 @Category(MediumTests.class)
85 public class TestStore extends TestCase {
86 public static final Log LOG = LogFactory.getLog(TestStore.class);
87
88 HStore store;
89 byte [] table = Bytes.toBytes("table");
90 byte [] family = Bytes.toBytes("family");
91
92 byte [] row = Bytes.toBytes("row");
93 byte [] row2 = Bytes.toBytes("row2");
94 byte [] qf1 = Bytes.toBytes("qf1");
95 byte [] qf2 = Bytes.toBytes("qf2");
96 byte [] qf3 = Bytes.toBytes("qf3");
97 byte [] qf4 = Bytes.toBytes("qf4");
98 byte [] qf5 = Bytes.toBytes("qf5");
99 byte [] qf6 = Bytes.toBytes("qf6");
100
101 NavigableSet<byte[]> qualifiers =
102 new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
103
104 List<Cell> expected = new ArrayList<Cell>();
105 List<Cell> result = new ArrayList<Cell>();
106
107 long id = System.currentTimeMillis();
108 Get get = new Get(row);
109
110 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
111 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
112
113
114
115
116
117
118 @Override
119 public void setUp() throws IOException {
120 qualifiers.add(qf1);
121 qualifiers.add(qf3);
122 qualifiers.add(qf5);
123
124 Iterator<byte[]> iter = qualifiers.iterator();
125 while(iter.hasNext()){
126 byte [] next = iter.next();
127 expected.add(new KeyValue(row, family, next, 1, (byte[])null));
128 get.addColumn(family, next);
129 }
130 }
131
132 private void init(String methodName) throws IOException {
133 init(methodName, HBaseConfiguration.create());
134 }
135
136 private void init(String methodName, Configuration conf)
137 throws IOException {
138 HColumnDescriptor hcd = new HColumnDescriptor(family);
139
140
141 hcd.setMaxVersions(4);
142 init(methodName, conf, hcd);
143 }
144
145 private void init(String methodName, Configuration conf,
146 HColumnDescriptor hcd) throws IOException {
147 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
148 init(methodName, conf, htd, hcd);
149 }
150
151 @SuppressWarnings("deprecation")
152 private void init(String methodName, Configuration conf, HTableDescriptor htd,
153 HColumnDescriptor hcd) throws IOException {
154
155 Path basedir = new Path(DIR+methodName);
156 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
157 String logName = "logs";
158 Path logdir = new Path(basedir, logName);
159
160 FileSystem fs = FileSystem.get(conf);
161
162 fs.delete(logdir, true);
163
164 htd.addFamily(hcd);
165 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
166 HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
167 HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
168
169 store = new HStore(region, hcd, conf);
170 }
171
172
173
174
175
176 public void testCreateWriter() throws Exception {
177 Configuration conf = HBaseConfiguration.create();
178 FileSystem fs = FileSystem.get(conf);
179
180 HColumnDescriptor hcd = new HColumnDescriptor(family);
181 hcd.setCompressionType(Compression.Algorithm.GZ);
182 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
183 init(getName(), conf, hcd);
184
185
186 StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
187 Path path = writer.getPath();
188 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
189 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
190 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
191 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
192 writer.close();
193
194
195 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
196 assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
197 assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
198 reader.close();
199 }
200
201 public void testDeleteExpiredStoreFiles() throws Exception {
202 int storeFileNum = 4;
203 int ttl = 4;
204 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
205 EnvironmentEdgeManagerTestHelper.injectEdge(edge);
206
207 Configuration conf = HBaseConfiguration.create();
208
209 conf.setBoolean("hbase.store.delete.expired.storefile", true);
210 HColumnDescriptor hcd = new HColumnDescriptor(family);
211 hcd.setTimeToLive(ttl);
212 init(getName(), conf, hcd);
213
214 long sleepTime = this.store.getScanInfo().getTtl() / storeFileNum;
215 long timeStamp;
216
217
218 for (int i = 1; i <= storeFileNum; i++) {
219 LOG.info("Adding some data for the store file #" + i);
220 timeStamp = EnvironmentEdgeManager.currentTimeMillis();
221 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
222 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
223 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
224 flush(i);
225 edge.incrementTime(sleepTime);
226 }
227
228
229 assertEquals(storeFileNum, this.store.getStorefiles().size());
230
231
232
233 for (int i = 1; i <= storeFileNum; i++) {
234
235 CompactionContext compaction = this.store.requestCompaction();
236 CompactionRequest cr = compaction.getRequest();
237
238
239 List<StoreFile> files = new ArrayList<StoreFile>(cr.getFiles());
240 assertEquals(Math.min(i, 2), cr.getFiles().size());
241 for (int j = 0; j < files.size(); j++) {
242 assertTrue(files.get(j).getReader().getMaxTimestamp() < (edge
243 .currentTimeMillis() - this.store.getScanInfo().getTtl()));
244 }
245
246
247 StoreFile compactedFile = this.store.compact(compaction).get(0);
248
249 assertEquals(0, compactedFile.getReader().getEntries());
250
251
252 edge.incrementTime(sleepTime);
253 }
254 }
255
256 public void testLowestModificationTime() throws Exception {
257 Configuration conf = HBaseConfiguration.create();
258 FileSystem fs = FileSystem.get(conf);
259
260 init(getName(), conf);
261
262 int storeFileNum = 4;
263 for (int i = 1; i <= storeFileNum; i++) {
264 LOG.info("Adding some data for the store file #"+i);
265 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
266 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
267 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
268 flush(i);
269 }
270
271 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
272 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
273 assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
274
275
276 store.compact(store.requestCompaction());
277 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
278 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
279 assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
280 }
281
282 private static long getLowestTimeStampFromFS(FileSystem fs,
283 final Collection<StoreFile> candidates) throws IOException {
284 long minTs = Long.MAX_VALUE;
285 if (candidates.isEmpty()) {
286 return minTs;
287 }
288 Path[] p = new Path[candidates.size()];
289 int i = 0;
290 for (StoreFile sf : candidates) {
291 p[i] = sf.getPath();
292 ++i;
293 }
294
295 FileStatus[] stats = fs.listStatus(p);
296 if (stats == null || stats.length == 0) {
297 return minTs;
298 }
299 for (FileStatus s : stats) {
300 minTs = Math.min(minTs, s.getModificationTime());
301 }
302 return minTs;
303 }
304
305
306
307
308
309 private static final int BLOCKSIZE_SMALL = 8192;
310
311
312
313
314 public void testEmptyStoreFile() throws IOException {
315 init(this.getName());
316
317 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
318 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
319 flush(1);
320
321
322 StoreFile f = this.store.getStorefiles().iterator().next();
323 Path storedir = f.getPath().getParent();
324 long seqid = f.getMaxSequenceId();
325 Configuration c = HBaseConfiguration.create();
326 FileSystem fs = FileSystem.get(c);
327 HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
328 StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
329 fs)
330 .withOutputDir(storedir)
331 .withFileContext(meta)
332 .build();
333 w.appendMetadata(seqid + 1, false);
334 w.close();
335 this.store.close();
336
337 this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
338 assertEquals(2, this.store.getStorefilesCount());
339
340 result = HBaseTestingUtility.getFromStoreFile(store,
341 get.getRow(),
342 qualifiers);
343 assertEquals(1, result.size());
344 }
345
346
347
348
349
350 public void testGet_FromMemStoreOnly() throws IOException {
351 init(this.getName());
352
353
354 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
355 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
356 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
357 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
358 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
359 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
360
361
362 result = HBaseTestingUtility.getFromStoreFile(store,
363 get.getRow(), qualifiers);
364
365
366 assertCheck();
367 }
368
369
370
371
372
373 public void testGet_FromFilesOnly() throws IOException {
374 init(this.getName());
375
376
377 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
378 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
379
380 flush(1);
381
382
383 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
384 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
385
386 flush(2);
387
388
389 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
390 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
391
392 flush(3);
393
394
395 result = HBaseTestingUtility.getFromStoreFile(store,
396 get.getRow(),
397 qualifiers);
398
399
400
401 Collections.sort(result, KeyValue.COMPARATOR);
402
403
404 assertCheck();
405 }
406
407
408
409
410
411 public void testGet_FromMemStoreAndFiles() throws IOException {
412 init(this.getName());
413
414
415 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
416 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
417
418 flush(1);
419
420
421 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
422 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
423
424 flush(2);
425
426
427 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
428 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
429
430
431 result = HBaseTestingUtility.getFromStoreFile(store,
432 get.getRow(), qualifiers);
433
434
435 Collections.sort(result, KeyValue.COMPARATOR);
436
437
438 assertCheck();
439 }
440
441 private void flush(int storeFilessize) throws IOException{
442 this.store.snapshot();
443 flushStore(store, id++);
444 assertEquals(storeFilessize, this.store.getStorefiles().size());
445 assertEquals(0, this.store.memstore.kvset.size());
446 }
447
448 private void assertCheck() {
449 assertEquals(expected.size(), result.size());
450 for(int i=0; i<expected.size(); i++) {
451 assertEquals(expected.get(i), result.get(i));
452 }
453 }
454
455
456
457
458
459
460
461 public void testIncrementColumnValue_ICVDuringFlush()
462 throws IOException, InterruptedException {
463 init(this.getName());
464
465 long oldValue = 1L;
466 long newValue = 3L;
467 this.store.add(new KeyValue(row, family, qf1,
468 System.currentTimeMillis(),
469 Bytes.toBytes(oldValue)));
470
471
472 this.store.snapshot();
473
474
475 this.store.add(new KeyValue(row, family, qf2,
476 System.currentTimeMillis(),
477 Bytes.toBytes(oldValue)));
478
479
480 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
481
482
483 assertTrue(ret > 0);
484
485
486 flushStore(store, id++);
487 assertEquals(1, this.store.getStorefiles().size());
488
489 assertEquals(2, this.store.memstore.kvset.size());
490
491
492 Get get = new Get(row);
493 get.addColumn(family, qf1);
494 get.setMaxVersions();
495 List<Cell> results = new ArrayList<Cell>();
496
497 results = HBaseTestingUtility.getFromStoreFile(store, get);
498 assertEquals(2, results.size());
499
500 long ts1 = results.get(0).getTimestamp();
501 long ts2 = results.get(1).getTimestamp();
502
503 assertTrue(ts1 > ts2);
504
505 assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
506 assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
507 }
508
509 @Override
510 protected void tearDown() throws Exception {
511 super.tearDown();
512 EnvironmentEdgeManagerTestHelper.reset();
513 }
514
515 public void testICV_negMemstoreSize() throws IOException {
516 init(this.getName());
517
518 long time = 100;
519 ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
520 ee.setValue(time);
521 EnvironmentEdgeManagerTestHelper.injectEdge(ee);
522 long newValue = 3L;
523 long size = 0;
524
525
526 size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
527 System.currentTimeMillis(),
528 Bytes.toBytes(newValue)));
529 size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
530 System.currentTimeMillis(),
531 Bytes.toBytes(newValue)));
532 size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
533 System.currentTimeMillis(),
534 Bytes.toBytes(newValue)));
535 size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
536 System.currentTimeMillis(),
537 Bytes.toBytes(newValue)));
538 size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
539 System.currentTimeMillis(),
540 Bytes.toBytes(newValue)));
541
542
543 for ( int i = 0 ; i < 10000 ; ++i) {
544 newValue++;
545
546 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
547 long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
548
549 if (ret != 0) System.out.println("ret: " + ret);
550 if (ret2 != 0) System.out.println("ret2: " + ret2);
551
552 assertTrue("ret: " + ret, ret >= 0);
553 size += ret;
554 assertTrue("ret2: " + ret2, ret2 >= 0);
555 size += ret2;
556
557
558 if (i % 1000 == 0)
559 ee.setValue(++time);
560 }
561
562 long computedSize=0;
563 for (KeyValue kv : this.store.memstore.kvset) {
564 long kvsize = MemStore.heapSizeChange(kv, true);
565
566 computedSize += kvsize;
567 }
568 assertEquals(computedSize, size);
569 }
570
571 public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
572 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
573 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
574 init(this.getName());
575
576 long oldValue = 1L;
577 long newValue = 3L;
578 this.store.add(new KeyValue(row, family, qf1,
579 EnvironmentEdgeManager.currentTimeMillis(),
580 Bytes.toBytes(oldValue)));
581
582
583 this.store.snapshot();
584
585
586 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
587
588
589 assertTrue(ret > 0);
590
591
592 flushStore(store, id++);
593 assertEquals(1, this.store.getStorefiles().size());
594 assertEquals(1, this.store.memstore.kvset.size());
595
596
597 newValue += 1;
598 this.store.updateColumnValue(row, family, qf1, newValue);
599
600
601 newValue += 1;
602 this.store.updateColumnValue(row, family, qf1, newValue);
603
604
605
606
607
608 Get get = new Get(row);
609 get.addColumn(family, qf1);
610 get.setMaxVersions();
611 List<Cell> results = new ArrayList<Cell>();
612
613 results = HBaseTestingUtility.getFromStoreFile(store, get);
614 assertEquals(2, results.size());
615
616 long ts1 = results.get(0).getTimestamp();
617 long ts2 = results.get(1).getTimestamp();
618
619 assertTrue(ts1 > ts2);
620 assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
621 assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
622
623 mee.setValue(2);
624 newValue += 1;
625 this.store.updateColumnValue(row, family, qf1, newValue);
626
627 results = HBaseTestingUtility.getFromStoreFile(store, get);
628 assertEquals(2, results.size());
629
630 ts1 = results.get(0).getTimestamp();
631 ts2 = results.get(1).getTimestamp();
632
633 assertTrue(ts1 > ts2);
634 assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
635 assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
636 }
637
638 public void testHandleErrorsInFlush() throws Exception {
639 LOG.info("Setting up a faulty file system that cannot write");
640
641 final Configuration conf = HBaseConfiguration.create();
642 User user = User.createUserForTesting(conf,
643 "testhandleerrorsinflush", new String[]{"foo"});
644
645 conf.setClass("fs.file.impl", FaultyFileSystem.class,
646 FileSystem.class);
647 user.runAs(new PrivilegedExceptionAction<Object>() {
648 public Object run() throws Exception {
649
650 FileSystem fs = FileSystem.get(conf);
651 assertEquals(FaultyFileSystem.class, fs.getClass());
652
653
654 init(getName(), conf);
655
656 LOG.info("Adding some data");
657 store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
658 store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
659 store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
660
661 LOG.info("Before flush, we should have no files");
662
663 Collection<StoreFileInfo> files =
664 store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
665 assertEquals(0, files != null ? files.size() : 0);
666
667
668 try {
669 LOG.info("Flushing");
670 flush(1);
671 fail("Didn't bubble up IOE!");
672 } catch (IOException ioe) {
673 assertTrue(ioe.getMessage().contains("Fault injected"));
674 }
675
676 LOG.info("After failed flush, we should still have no files!");
677 files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
678 assertEquals(0, files != null ? files.size() : 0);
679 return null;
680 }
681 });
682 }
683
684
685 static class FaultyFileSystem extends FilterFileSystem {
686 List<SoftReference<FaultyOutputStream>> outStreams =
687 new ArrayList<SoftReference<FaultyOutputStream>>();
688 private long faultPos = 200;
689
690 public FaultyFileSystem() {
691 super(new LocalFileSystem());
692 System.err.println("Creating faulty!");
693 }
694
695 @Override
696 public FSDataOutputStream create(Path p) throws IOException {
697 return new FaultyOutputStream(super.create(p), faultPos);
698 }
699
700 @Override
701 public FSDataOutputStream create(Path f, FsPermission permission,
702 boolean overwrite, int bufferSize, short replication, long blockSize,
703 Progressable progress) throws IOException {
704 return new FaultyOutputStream(super.create(f, permission,
705 overwrite, bufferSize, replication, blockSize, progress), faultPos);
706 }
707
708 public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
709 int bufferSize, short replication, long blockSize, Progressable progress)
710 throws IOException {
711
712
713 return create(f, overwrite, bufferSize, replication, blockSize, progress);
714 }
715 }
716
717 static class FaultyOutputStream extends FSDataOutputStream {
718 volatile long faultPos = Long.MAX_VALUE;
719
720 public FaultyOutputStream(FSDataOutputStream out,
721 long faultPos) throws IOException {
722 super(out, null);
723 this.faultPos = faultPos;
724 }
725
726 @Override
727 public void write(byte[] buf, int offset, int length) throws IOException {
728 System.err.println("faulty stream write at pos " + getPos());
729 injectFault();
730 super.write(buf, offset, length);
731 }
732
733 private void injectFault() throws IOException {
734 if (getPos() >= faultPos) {
735 throw new IOException("Fault injected");
736 }
737 }
738 }
739
740
741
742 private static void flushStore(HStore store, long id) throws IOException {
743 StoreFlushContext storeFlushCtx = store.createFlushContext(id);
744 storeFlushCtx.prepare();
745 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
746 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
747 }
748
749
750
751
752
753
754
755
756
757
758
759 List<Cell> getKeyValueSet(long[] timestamps, int numRows,
760 byte[] qualifier, byte[] family) {
761 List<Cell> kvList = new ArrayList<Cell>();
762 for (int i=1;i<=numRows;i++) {
763 byte[] b = Bytes.toBytes(i);
764 for (long timestamp: timestamps) {
765 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
766 }
767 }
768 return kvList;
769 }
770
771
772
773
774
775 public void testMultipleTimestamps() throws IOException {
776 int numRows = 1;
777 long[] timestamps1 = new long[] {1,5,10,20};
778 long[] timestamps2 = new long[] {30,80};
779
780 init(this.getName());
781
782 List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
783 for (Cell kv : kvList1) {
784 this.store.add(KeyValueUtil.ensureKeyValue(kv));
785 }
786
787 this.store.snapshot();
788 flushStore(store, id++);
789
790 List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
791 for(Cell kv : kvList2) {
792 this.store.add(KeyValueUtil.ensureKeyValue(kv));
793 }
794
795 List<Cell> result;
796 Get get = new Get(Bytes.toBytes(1));
797 get.addColumn(family,qf1);
798
799 get.setTimeRange(0,15);
800 result = HBaseTestingUtility.getFromStoreFile(store, get);
801 assertTrue(result.size()>0);
802
803 get.setTimeRange(40,90);
804 result = HBaseTestingUtility.getFromStoreFile(store, get);
805 assertTrue(result.size()>0);
806
807 get.setTimeRange(10,45);
808 result = HBaseTestingUtility.getFromStoreFile(store, get);
809 assertTrue(result.size()>0);
810
811 get.setTimeRange(80,145);
812 result = HBaseTestingUtility.getFromStoreFile(store, get);
813 assertTrue(result.size()>0);
814
815 get.setTimeRange(1,2);
816 result = HBaseTestingUtility.getFromStoreFile(store, get);
817 assertTrue(result.size()>0);
818
819 get.setTimeRange(90,200);
820 result = HBaseTestingUtility.getFromStoreFile(store, get);
821 assertTrue(result.size()==0);
822 }
823
824
825
826
827
828
829 public void testSplitWithEmptyColFam() throws IOException {
830 init(this.getName());
831 assertNull(store.getSplitPoint());
832 store.getHRegion().forceSplit(null);
833 assertNull(store.getSplitPoint());
834 store.getHRegion().clearSplit_TESTS_ONLY();
835 }
836
837 public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
838 final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
839 long anyValue = 10;
840
841
842
843
844 Configuration conf = HBaseConfiguration.create();
845 conf.setLong(CONFIG_KEY, anyValue);
846 init(getName() + "-xml", conf);
847 assertTrue(store.throttleCompaction(anyValue + 1));
848 assertFalse(store.throttleCompaction(anyValue));
849
850
851 --anyValue;
852 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
853 HColumnDescriptor hcd = new HColumnDescriptor(family);
854 htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
855 init(getName() + "-htd", conf, htd, hcd);
856 assertTrue(store.throttleCompaction(anyValue + 1));
857 assertFalse(store.throttleCompaction(anyValue));
858
859
860 --anyValue;
861 hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
862 init(getName() + "-hcd", conf, htd, hcd);
863 assertTrue(store.throttleCompaction(anyValue + 1));
864 assertFalse(store.throttleCompaction(anyValue));
865 }
866
867 public static class DummyStoreEngine extends DefaultStoreEngine {
868 public static DefaultCompactor lastCreatedCompactor = null;
869 @Override
870 protected void createComponents(
871 Configuration conf, Store store, KVComparator comparator) throws IOException {
872 super.createComponents(conf, store, comparator);
873 lastCreatedCompactor = this.compactor;
874 }
875 }
876
877 public void testStoreUsesSearchEngineOverride() throws Exception {
878 Configuration conf = HBaseConfiguration.create();
879 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
880 init(this.getName(), conf);
881 assertEquals(DummyStoreEngine.lastCreatedCompactor, this.store.storeEngine.getCompactor());
882 }
883 }
884