1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.lang.ref.SoftReference;
24 import java.security.PrivilegedExceptionAction;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.NavigableSet;
31 import java.util.concurrent.ConcurrentSkipListSet;
32 import java.util.concurrent.atomic.AtomicBoolean;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.FilterFileSystem;
41 import org.apache.hadoop.fs.LocalFileSystem;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.fs.permission.FsPermission;
44 import org.apache.hadoop.hbase.Cell;
45 import org.apache.hadoop.hbase.CellUtil;
46 import org.apache.hadoop.hbase.HBaseConfiguration;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HColumnDescriptor;
49 import org.apache.hadoop.hbase.HRegionInfo;
50 import org.apache.hadoop.hbase.HTableDescriptor;
51 import org.apache.hadoop.hbase.KeyValue;
52 import org.apache.hadoop.hbase.KeyValue.KVComparator;
53 import org.apache.hadoop.hbase.KeyValueUtil;
54 import org.apache.hadoop.hbase.MediumTests;
55 import org.apache.hadoop.hbase.TableName;
56 import org.apache.hadoop.hbase.client.Get;
57 import org.apache.hadoop.hbase.io.compress.Compression;
58 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
59 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
60 import org.apache.hadoop.hbase.io.hfile.HFile;
61 import org.apache.hadoop.hbase.io.hfile.HFileContext;
62 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
63 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
64 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
65 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
66 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
67 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
68 import org.apache.hadoop.hbase.regionserver.wal.HLog;
69 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
70 import org.apache.hadoop.hbase.security.User;
71 import org.apache.hadoop.hbase.util.Bytes;
72 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
74 import org.apache.hadoop.hbase.util.FSUtils;
75 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
76 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
77 import org.apache.hadoop.util.Progressable;
78 import org.junit.After;
79 import org.junit.Assert;
80 import org.junit.Before;
81 import org.junit.Rule;
82 import org.junit.Test;
83 import org.junit.experimental.categories.Category;
84 import org.junit.rules.TestName;
85 import org.mockito.Mockito;
86
87
88
89
90 @Category(MediumTests.class)
91 public class TestStore {
92 public static final Log LOG = LogFactory.getLog(TestStore.class);
93 @Rule public TestName name = new TestName();
94
95 HStore store;
96 byte [] table = Bytes.toBytes("table");
97 byte [] family = Bytes.toBytes("family");
98
99 byte [] row = Bytes.toBytes("row");
100 byte [] row2 = Bytes.toBytes("row2");
101 byte [] qf1 = Bytes.toBytes("qf1");
102 byte [] qf2 = Bytes.toBytes("qf2");
103 byte [] qf3 = Bytes.toBytes("qf3");
104 byte [] qf4 = Bytes.toBytes("qf4");
105 byte [] qf5 = Bytes.toBytes("qf5");
106 byte [] qf6 = Bytes.toBytes("qf6");
107
108 NavigableSet<byte[]> qualifiers =
109 new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
110
111 List<Cell> expected = new ArrayList<Cell>();
112 List<Cell> result = new ArrayList<Cell>();
113
114 long id = System.currentTimeMillis();
115 Get get = new Get(row);
116
117 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
118 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
119
120
121
122
123
124
125 @Before
126 public void setUp() throws IOException {
127 qualifiers.add(qf1);
128 qualifiers.add(qf3);
129 qualifiers.add(qf5);
130
131 Iterator<byte[]> iter = qualifiers.iterator();
132 while(iter.hasNext()){
133 byte [] next = iter.next();
134 expected.add(new KeyValue(row, family, next, 1, (byte[])null));
135 get.addColumn(family, next);
136 }
137 }
138
139 private void init(String methodName) throws IOException {
140 init(methodName, HBaseConfiguration.create());
141 }
142
143 private void init(String methodName, Configuration conf)
144 throws IOException {
145 HColumnDescriptor hcd = new HColumnDescriptor(family);
146
147
148 hcd.setMaxVersions(4);
149 init(methodName, conf, hcd);
150 }
151
152 private void init(String methodName, Configuration conf,
153 HColumnDescriptor hcd) throws IOException {
154 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
155 init(methodName, conf, htd, hcd);
156 }
157
158 @SuppressWarnings("deprecation")
159 private Store init(String methodName, Configuration conf, HTableDescriptor htd,
160 HColumnDescriptor hcd) throws IOException {
161
162 Path basedir = new Path(DIR+methodName);
163 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
164 String logName = "logs";
165 Path logdir = new Path(basedir, logName);
166
167 FileSystem fs = FileSystem.get(conf);
168
169 fs.delete(logdir, true);
170
171 htd.addFamily(hcd);
172 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
173 HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
174 HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
175
176 store = new HStore(region, hcd, conf);
177 return store;
178 }
179
180
181
182
183
184
185 @Test
186 public void testFlushSizeAccounting() throws Exception {
187 LOG.info("Setting up a faulty file system that cannot write in " +
188 this.name.getMethodName());
189 final Configuration conf = HBaseConfiguration.create();
190
191 conf.setInt("hbase.hstore.flush.retries.number", 1);
192 User user = User.createUserForTesting(conf, this.name.getMethodName(),
193 new String[]{"foo"});
194
195 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
196 user.runAs(new PrivilegedExceptionAction<Object>() {
197 public Object run() throws Exception {
198
199 FileSystem fs = FileSystem.get(conf);
200 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
201 FaultyFileSystem ffs = (FaultyFileSystem)fs;
202
203
204 init(name.getMethodName(), conf);
205
206 long size = store.memstore.getFlushableSize();
207 Assert.assertEquals(0, size);
208 LOG.info("Adding some data");
209 long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
210 size = store.memstore.getFlushableSize();
211 Assert.assertEquals(kvSize, size);
212
213 try {
214 LOG.info("Flushing");
215 flushStore(store, id++);
216 Assert.fail("Didn't bubble up IOE!");
217 } catch (IOException ioe) {
218 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
219 }
220 size = store.memstore.getFlushableSize();
221 Assert.assertEquals(kvSize, size);
222 store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
223
224
225 Assert.assertEquals(kvSize, size);
226 ffs.fault.set(false);
227 flushStore(store, id++);
228 size = store.memstore.getFlushableSize();
229
230 Assert.assertEquals(kvSize, size);
231 flushStore(store, id++);
232 size = store.memstore.getFlushableSize();
233 Assert.assertEquals(0, size);
234 return null;
235 }
236 });
237 }
238
239
240
241
242
243 @Test
244 public void testCreateWriter() throws Exception {
245 Configuration conf = HBaseConfiguration.create();
246 FileSystem fs = FileSystem.get(conf);
247
248 HColumnDescriptor hcd = new HColumnDescriptor(family);
249 hcd.setCompressionType(Compression.Algorithm.GZ);
250 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
251 init(name.getMethodName(), conf, hcd);
252
253
254 StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
255 Path path = writer.getPath();
256 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
257 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
258 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
259 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
260 writer.close();
261
262
263 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
264 Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
265 Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
266 reader.close();
267 }
268
269 @Test
270 public void testDeleteExpiredStoreFiles() throws Exception {
271 testDeleteExpiredStoreFiles(0);
272 testDeleteExpiredStoreFiles(1);
273 }
274
275
276
277
278 public void testDeleteExpiredStoreFiles(int minVersions) throws Exception {
279 int storeFileNum = 4;
280 int ttl = 4;
281 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
282 EnvironmentEdgeManagerTestHelper.injectEdge(edge);
283
284 Configuration conf = HBaseConfiguration.create();
285
286 conf.setBoolean("hbase.store.delete.expired.storefile", true);
287
288 conf.setInt(CompactionConfiguration.MIN_KEY, 5);
289
290 HColumnDescriptor hcd = new HColumnDescriptor(family);
291 hcd.setMinVersions(minVersions);
292 hcd.setTimeToLive(ttl);
293 init(name.getMethodName(), conf, hcd);
294
295 long storeTtl = this.store.getScanInfo().getTtl();
296 long sleepTime = storeTtl / storeFileNum;
297 long timeStamp;
298
299
300 for (int i = 1; i <= storeFileNum; i++) {
301 LOG.info("Adding some data for the store file #" + i);
302 timeStamp = EnvironmentEdgeManager.currentTimeMillis();
303 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
304 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
305 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
306 flush(i);
307 edge.incrementTime(sleepTime);
308 }
309
310
311 Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
312
313
314
315 for (int i = 1; i <= storeFileNum - 1; i++) {
316
317 Assert.assertNull(this.store.requestCompaction());
318 Collection<StoreFile> sfs = this.store.getStorefiles();
319
320 if (minVersions == 0) {
321 Assert.assertEquals(storeFileNum - i, sfs.size());
322
323 for (StoreFile sf : sfs) {
324 Assert.assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTimeMillis() - storeTtl));
325 }
326 } else {
327 Assert.assertEquals(storeFileNum, sfs.size());
328 }
329
330 edge.incrementTime(sleepTime);
331 }
332 Assert.assertNull(this.store.requestCompaction());
333 Collection<StoreFile> sfs = this.store.getStorefiles();
334
335 if (minVersions == 0) {
336 Assert.assertEquals(1, sfs.size());
337 }
338 long ts = sfs.iterator().next().getReader().getMaxTimestamp();
339 Assert.assertTrue(ts < (edge.currentTimeMillis() - storeTtl));
340 }
341
342 @Test
343 public void testLowestModificationTime() throws Exception {
344 Configuration conf = HBaseConfiguration.create();
345 FileSystem fs = FileSystem.get(conf);
346
347 init(name.getMethodName(), conf);
348
349 int storeFileNum = 4;
350 for (int i = 1; i <= storeFileNum; i++) {
351 LOG.info("Adding some data for the store file #"+i);
352 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
353 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
354 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
355 flush(i);
356 }
357
358 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
359 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
360 Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
361
362
363 store.compact(store.requestCompaction());
364 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
365 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
366 Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
367 }
368
369 private static long getLowestTimeStampFromFS(FileSystem fs,
370 final Collection<StoreFile> candidates) throws IOException {
371 long minTs = Long.MAX_VALUE;
372 if (candidates.isEmpty()) {
373 return minTs;
374 }
375 Path[] p = new Path[candidates.size()];
376 int i = 0;
377 for (StoreFile sf : candidates) {
378 p[i] = sf.getPath();
379 ++i;
380 }
381
382 FileStatus[] stats = fs.listStatus(p);
383 if (stats == null || stats.length == 0) {
384 return minTs;
385 }
386 for (FileStatus s : stats) {
387 minTs = Math.min(minTs, s.getModificationTime());
388 }
389 return minTs;
390 }
391
392
393
394
395
396 private static final int BLOCKSIZE_SMALL = 8192;
397
398
399
400
401 @Test
402 public void testEmptyStoreFile() throws IOException {
403 init(this.name.getMethodName());
404
405 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
406 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
407 flush(1);
408
409
410 StoreFile f = this.store.getStorefiles().iterator().next();
411 Path storedir = f.getPath().getParent();
412 long seqid = f.getMaxSequenceId();
413 Configuration c = HBaseConfiguration.create();
414 FileSystem fs = FileSystem.get(c);
415 HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
416 StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
417 fs)
418 .withOutputDir(storedir)
419 .withFileContext(meta)
420 .build();
421 w.appendMetadata(seqid + 1, false);
422 w.close();
423 this.store.close();
424
425 this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
426 Assert.assertEquals(2, this.store.getStorefilesCount());
427
428 result = HBaseTestingUtility.getFromStoreFile(store,
429 get.getRow(),
430 qualifiers);
431 Assert.assertEquals(1, result.size());
432 }
433
434
435
436
437
438 @Test
439 public void testGet_FromMemStoreOnly() throws IOException {
440 init(this.name.getMethodName());
441
442
443 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
444 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
445 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
446 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
447 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
448 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
449
450
451 result = HBaseTestingUtility.getFromStoreFile(store,
452 get.getRow(), qualifiers);
453
454
455 assertCheck();
456 }
457
458
459
460
461
462 @Test
463 public void testGet_FromFilesOnly() throws IOException {
464 init(this.name.getMethodName());
465
466
467 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
468 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
469
470 flush(1);
471
472
473 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
474 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
475
476 flush(2);
477
478
479 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
480 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
481
482 flush(3);
483
484
485 result = HBaseTestingUtility.getFromStoreFile(store,
486 get.getRow(),
487 qualifiers);
488
489
490
491 Collections.sort(result, KeyValue.COMPARATOR);
492
493
494 assertCheck();
495 }
496
497
498
499
500
501 @Test
502 public void testGet_FromMemStoreAndFiles() throws IOException {
503 init(this.name.getMethodName());
504
505
506 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
507 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
508
509 flush(1);
510
511
512 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
513 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
514
515 flush(2);
516
517
518 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
519 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
520
521
522 result = HBaseTestingUtility.getFromStoreFile(store,
523 get.getRow(), qualifiers);
524
525
526 Collections.sort(result, KeyValue.COMPARATOR);
527
528
529 assertCheck();
530 }
531
532 private void flush(int storeFilessize) throws IOException{
533 this.store.snapshot();
534 flushStore(store, id++);
535 Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
536 Assert.assertEquals(0, this.store.memstore.kvset.size());
537 }
538
539 private void assertCheck() {
540 Assert.assertEquals(expected.size(), result.size());
541 for(int i=0; i<expected.size(); i++) {
542 Assert.assertEquals(expected.get(i), result.get(i));
543 }
544 }
545
546
547
548
549
550
551
552 @Test
553 public void testIncrementColumnValue_ICVDuringFlush()
554 throws IOException, InterruptedException {
555 init(this.name.getMethodName());
556
557 long oldValue = 1L;
558 long newValue = 3L;
559 this.store.add(new KeyValue(row, family, qf1,
560 System.currentTimeMillis(),
561 Bytes.toBytes(oldValue)));
562
563
564 this.store.snapshot();
565
566
567 this.store.add(new KeyValue(row, family, qf2,
568 System.currentTimeMillis(),
569 Bytes.toBytes(oldValue)));
570
571
572 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
573
574
575 Assert.assertTrue(ret > 0);
576
577
578 flushStore(store, id++);
579 Assert.assertEquals(1, this.store.getStorefiles().size());
580
581 Assert.assertEquals(2, this.store.memstore.kvset.size());
582
583
584 Get get = new Get(row);
585 get.addColumn(family, qf1);
586 get.setMaxVersions();
587 List<Cell> results = new ArrayList<Cell>();
588
589 results = HBaseTestingUtility.getFromStoreFile(store, get);
590 Assert.assertEquals(2, results.size());
591
592 long ts1 = results.get(0).getTimestamp();
593 long ts2 = results.get(1).getTimestamp();
594
595 Assert.assertTrue(ts1 > ts2);
596
597 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
598 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
599 }
600
601 @After
602 public void tearDown() throws Exception {
603 EnvironmentEdgeManagerTestHelper.reset();
604 }
605
606 @Test
607 public void testICV_negMemstoreSize() throws IOException {
608 init(this.name.getMethodName());
609
610 long time = 100;
611 ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
612 ee.setValue(time);
613 EnvironmentEdgeManagerTestHelper.injectEdge(ee);
614 long newValue = 3L;
615 long size = 0;
616
617
618 size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
619 System.currentTimeMillis(),
620 Bytes.toBytes(newValue)));
621 size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
622 System.currentTimeMillis(),
623 Bytes.toBytes(newValue)));
624 size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
625 System.currentTimeMillis(),
626 Bytes.toBytes(newValue)));
627 size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
628 System.currentTimeMillis(),
629 Bytes.toBytes(newValue)));
630 size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
631 System.currentTimeMillis(),
632 Bytes.toBytes(newValue)));
633
634
635 for ( int i = 0 ; i < 10000 ; ++i) {
636 newValue++;
637
638 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
639 long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
640
641 if (ret != 0) System.out.println("ret: " + ret);
642 if (ret2 != 0) System.out.println("ret2: " + ret2);
643
644 Assert.assertTrue("ret: " + ret, ret >= 0);
645 size += ret;
646 Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
647 size += ret2;
648
649
650 if (i % 1000 == 0)
651 ee.setValue(++time);
652 }
653
654 long computedSize=0;
655 for (KeyValue kv : this.store.memstore.kvset) {
656 long kvsize = MemStore.heapSizeChange(kv, true);
657
658 computedSize += kvsize;
659 }
660 Assert.assertEquals(computedSize, size);
661 }
662
663 @Test
664 public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
665 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
666 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
667 init(this.name.getMethodName());
668
669 long oldValue = 1L;
670 long newValue = 3L;
671 this.store.add(new KeyValue(row, family, qf1,
672 EnvironmentEdgeManager.currentTimeMillis(),
673 Bytes.toBytes(oldValue)));
674
675
676 this.store.snapshot();
677
678
679 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
680
681
682 Assert.assertTrue(ret > 0);
683
684
685 flushStore(store, id++);
686 Assert.assertEquals(1, this.store.getStorefiles().size());
687 Assert.assertEquals(1, this.store.memstore.kvset.size());
688
689
690 newValue += 1;
691 this.store.updateColumnValue(row, family, qf1, newValue);
692
693
694 newValue += 1;
695 this.store.updateColumnValue(row, family, qf1, newValue);
696
697
698
699
700
701 Get get = new Get(row);
702 get.addColumn(family, qf1);
703 get.setMaxVersions();
704 List<Cell> results = new ArrayList<Cell>();
705
706 results = HBaseTestingUtility.getFromStoreFile(store, get);
707 Assert.assertEquals(2, results.size());
708
709 long ts1 = results.get(0).getTimestamp();
710 long ts2 = results.get(1).getTimestamp();
711
712 Assert.assertTrue(ts1 > ts2);
713 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
714 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
715
716 mee.setValue(2);
717 newValue += 1;
718 this.store.updateColumnValue(row, family, qf1, newValue);
719
720 results = HBaseTestingUtility.getFromStoreFile(store, get);
721 Assert.assertEquals(2, results.size());
722
723 ts1 = results.get(0).getTimestamp();
724 ts2 = results.get(1).getTimestamp();
725
726 Assert.assertTrue(ts1 > ts2);
727 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
728 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
729 }
730
731 @Test
732 public void testHandleErrorsInFlush() throws Exception {
733 LOG.info("Setting up a faulty file system that cannot write");
734
735 final Configuration conf = HBaseConfiguration.create();
736 User user = User.createUserForTesting(conf,
737 "testhandleerrorsinflush", new String[]{"foo"});
738
739 conf.setClass("fs.file.impl", FaultyFileSystem.class,
740 FileSystem.class);
741 user.runAs(new PrivilegedExceptionAction<Object>() {
742 public Object run() throws Exception {
743
744 FileSystem fs = FileSystem.get(conf);
745 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
746
747
748 init(name.getMethodName(), conf);
749
750 LOG.info("Adding some data");
751 store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
752 store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
753 store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
754
755 LOG.info("Before flush, we should have no files");
756
757 Collection<StoreFileInfo> files =
758 store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
759 Assert.assertEquals(0, files != null ? files.size() : 0);
760
761
762 try {
763 LOG.info("Flushing");
764 flush(1);
765 Assert.fail("Didn't bubble up IOE!");
766 } catch (IOException ioe) {
767 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
768 }
769
770 LOG.info("After failed flush, we should still have no files!");
771 files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
772 Assert.assertEquals(0, files != null ? files.size() : 0);
773 store.getHRegion().getLog().closeAndDelete();
774 return null;
775 }
776 });
777 FileSystem.closeAllForUGI(user.getUGI());
778 }
779
780
781
782
783
784 static class FaultyFileSystem extends FilterFileSystem {
785 List<SoftReference<FaultyOutputStream>> outStreams =
786 new ArrayList<SoftReference<FaultyOutputStream>>();
787 private long faultPos = 200;
788 AtomicBoolean fault = new AtomicBoolean(true);
789
790 public FaultyFileSystem() {
791 super(new LocalFileSystem());
792 System.err.println("Creating faulty!");
793 }
794
795 @Override
796 public FSDataOutputStream create(Path p) throws IOException {
797 return new FaultyOutputStream(super.create(p), faultPos, fault);
798 }
799
800 @Override
801 public FSDataOutputStream create(Path f, FsPermission permission,
802 boolean overwrite, int bufferSize, short replication, long blockSize,
803 Progressable progress) throws IOException {
804 return new FaultyOutputStream(super.create(f, permission,
805 overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
806 }
807
808 public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
809 int bufferSize, short replication, long blockSize, Progressable progress)
810 throws IOException {
811
812
813 return create(f, overwrite, bufferSize, replication, blockSize, progress);
814 }
815 }
816
817 static class FaultyOutputStream extends FSDataOutputStream {
818 volatile long faultPos = Long.MAX_VALUE;
819 private final AtomicBoolean fault;
820
821 public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
822 throws IOException {
823 super(out, null);
824 this.faultPos = faultPos;
825 this.fault = fault;
826 }
827
828 @Override
829 public void write(byte[] buf, int offset, int length) throws IOException {
830 System.err.println("faulty stream write at pos " + getPos());
831 injectFault();
832 super.write(buf, offset, length);
833 }
834
835 private void injectFault() throws IOException {
836 if (this.fault.get() && getPos() >= faultPos) {
837 throw new IOException("Fault injected");
838 }
839 }
840 }
841
842 private static void flushStore(HStore store, long id) throws IOException {
843 StoreFlushContext storeFlushCtx = store.createFlushContext(id);
844 storeFlushCtx.prepare();
845 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
846 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
847 }
848
849
850
851
852
853
854
855
856
857 List<Cell> getKeyValueSet(long[] timestamps, int numRows,
858 byte[] qualifier, byte[] family) {
859 List<Cell> kvList = new ArrayList<Cell>();
860 for (int i=1;i<=numRows;i++) {
861 byte[] b = Bytes.toBytes(i);
862 for (long timestamp: timestamps) {
863 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
864 }
865 }
866 return kvList;
867 }
868
869
870
871
872
873 @Test
874 public void testMultipleTimestamps() throws IOException {
875 int numRows = 1;
876 long[] timestamps1 = new long[] {1,5,10,20};
877 long[] timestamps2 = new long[] {30,80};
878
879 init(this.name.getMethodName());
880
881 List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
882 for (Cell kv : kvList1) {
883 this.store.add(KeyValueUtil.ensureKeyValue(kv));
884 }
885
886 this.store.snapshot();
887 flushStore(store, id++);
888
889 List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
890 for(Cell kv : kvList2) {
891 this.store.add(KeyValueUtil.ensureKeyValue(kv));
892 }
893
894 List<Cell> result;
895 Get get = new Get(Bytes.toBytes(1));
896 get.addColumn(family,qf1);
897
898 get.setTimeRange(0,15);
899 result = HBaseTestingUtility.getFromStoreFile(store, get);
900 Assert.assertTrue(result.size()>0);
901
902 get.setTimeRange(40,90);
903 result = HBaseTestingUtility.getFromStoreFile(store, get);
904 Assert.assertTrue(result.size()>0);
905
906 get.setTimeRange(10,45);
907 result = HBaseTestingUtility.getFromStoreFile(store, get);
908 Assert.assertTrue(result.size()>0);
909
910 get.setTimeRange(80,145);
911 result = HBaseTestingUtility.getFromStoreFile(store, get);
912 Assert.assertTrue(result.size()>0);
913
914 get.setTimeRange(1,2);
915 result = HBaseTestingUtility.getFromStoreFile(store, get);
916 Assert.assertTrue(result.size()>0);
917
918 get.setTimeRange(90,200);
919 result = HBaseTestingUtility.getFromStoreFile(store, get);
920 Assert.assertTrue(result.size()==0);
921 }
922
923
924
925
926
927
928 @Test
929 public void testSplitWithEmptyColFam() throws IOException {
930 init(this.name.getMethodName());
931 Assert.assertNull(store.getSplitPoint());
932 store.getHRegion().forceSplit(null);
933 Assert.assertNull(store.getSplitPoint());
934 store.getHRegion().clearSplit();
935 }
936
937 @Test
938 public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
939 final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
940 long anyValue = 10;
941
942
943
944
945 Configuration conf = HBaseConfiguration.create();
946 conf.setLong(CONFIG_KEY, anyValue);
947 init(name.getMethodName() + "-xml", conf);
948 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
949 Assert.assertFalse(store.throttleCompaction(anyValue));
950
951
952 --anyValue;
953 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
954 HColumnDescriptor hcd = new HColumnDescriptor(family);
955 htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
956 init(name.getMethodName() + "-htd", conf, htd, hcd);
957 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
958 Assert.assertFalse(store.throttleCompaction(anyValue));
959
960
961 --anyValue;
962 hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
963 init(name.getMethodName() + "-hcd", conf, htd, hcd);
964 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
965 Assert.assertFalse(store.throttleCompaction(anyValue));
966 }
967
968 public static class DummyStoreEngine extends DefaultStoreEngine {
969 public static DefaultCompactor lastCreatedCompactor = null;
970 @Override
971 protected void createComponents(
972 Configuration conf, Store store, KVComparator comparator) throws IOException {
973 super.createComponents(conf, store, comparator);
974 lastCreatedCompactor = this.compactor;
975 }
976 }
977
978 @Test
979 public void testStoreUsesSearchEngineOverride() throws Exception {
980 Configuration conf = HBaseConfiguration.create();
981 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
982 init(this.name.getMethodName(), conf);
983 Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
984 this.store.storeEngine.getCompactor());
985 }
986 }