1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.lang.ref.SoftReference;
24 import java.security.PrivilegedExceptionAction;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.NavigableSet;
31 import java.util.concurrent.ConcurrentSkipListSet;
32 import java.util.concurrent.atomic.AtomicBoolean;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.FilterFileSystem;
41 import org.apache.hadoop.fs.LocalFileSystem;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.fs.permission.FsPermission;
44 import org.apache.hadoop.hbase.Cell;
45 import org.apache.hadoop.hbase.CellUtil;
46 import org.apache.hadoop.hbase.HBaseConfiguration;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HColumnDescriptor;
49 import org.apache.hadoop.hbase.HRegionInfo;
50 import org.apache.hadoop.hbase.HTableDescriptor;
51 import org.apache.hadoop.hbase.KeyValue;
52 import org.apache.hadoop.hbase.KeyValue.KVComparator;
53 import org.apache.hadoop.hbase.KeyValueUtil;
54 import org.apache.hadoop.hbase.MediumTests;
55 import org.apache.hadoop.hbase.TableName;
56 import org.apache.hadoop.hbase.client.Get;
57 import org.apache.hadoop.hbase.io.compress.Compression;
58 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
59 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
60 import org.apache.hadoop.hbase.io.hfile.HFile;
61 import org.apache.hadoop.hbase.io.hfile.HFileContext;
62 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
63 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
64 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
65 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
66 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
67 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
68 import org.apache.hadoop.hbase.regionserver.wal.HLog;
69 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
70 import org.apache.hadoop.hbase.security.User;
71 import org.apache.hadoop.hbase.util.Bytes;
72 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
74 import org.apache.hadoop.hbase.util.FSUtils;
75 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
76 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
77 import org.apache.hadoop.util.Progressable;
78 import org.junit.After;
79 import org.junit.Assert;
80 import org.junit.Before;
81 import org.junit.Rule;
82 import org.junit.Test;
83 import org.junit.experimental.categories.Category;
84 import org.junit.rules.TestName;
85 import org.mockito.Mockito;
86
87
88
89
90 @Category(MediumTests.class)
91 public class TestStore {
92 public static final Log LOG = LogFactory.getLog(TestStore.class);
93 @Rule public TestName name = new TestName();
94
95 HStore store;
96 byte [] table = Bytes.toBytes("table");
97 byte [] family = Bytes.toBytes("family");
98
99 byte [] row = Bytes.toBytes("row");
100 byte [] row2 = Bytes.toBytes("row2");
101 byte [] qf1 = Bytes.toBytes("qf1");
102 byte [] qf2 = Bytes.toBytes("qf2");
103 byte [] qf3 = Bytes.toBytes("qf3");
104 byte [] qf4 = Bytes.toBytes("qf4");
105 byte [] qf5 = Bytes.toBytes("qf5");
106 byte [] qf6 = Bytes.toBytes("qf6");
107
108 NavigableSet<byte[]> qualifiers =
109 new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
110
111 List<Cell> expected = new ArrayList<Cell>();
112 List<Cell> result = new ArrayList<Cell>();
113
114 long id = System.currentTimeMillis();
115 Get get = new Get(row);
116
117 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
118 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
119
120
121
122
123
124
125 @Before
126 public void setUp() throws IOException {
127 qualifiers.add(qf1);
128 qualifiers.add(qf3);
129 qualifiers.add(qf5);
130
131 Iterator<byte[]> iter = qualifiers.iterator();
132 while(iter.hasNext()){
133 byte [] next = iter.next();
134 expected.add(new KeyValue(row, family, next, 1, (byte[])null));
135 get.addColumn(family, next);
136 }
137 }
138
139 private void init(String methodName) throws IOException {
140 init(methodName, HBaseConfiguration.create());
141 }
142
143 private void init(String methodName, Configuration conf)
144 throws IOException {
145 HColumnDescriptor hcd = new HColumnDescriptor(family);
146
147
148 hcd.setMaxVersions(4);
149 init(methodName, conf, hcd);
150 }
151
152 private void init(String methodName, Configuration conf,
153 HColumnDescriptor hcd) throws IOException {
154 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
155 init(methodName, conf, htd, hcd);
156 }
157
158 @SuppressWarnings("deprecation")
159 private Store init(String methodName, Configuration conf, HTableDescriptor htd,
160 HColumnDescriptor hcd) throws IOException {
161
162 Path basedir = new Path(DIR+methodName);
163 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
164 String logName = "logs";
165 Path logdir = new Path(basedir, logName);
166
167 FileSystem fs = FileSystem.get(conf);
168
169 fs.delete(logdir, true);
170
171 htd.addFamily(hcd);
172 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
173 HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
174 HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
175
176 store = new HStore(region, hcd, conf);
177 return store;
178 }
179
180
181
182
183
184
185 @Test
186 public void testFlushSizeAccounting() throws Exception {
187 LOG.info("Setting up a faulty file system that cannot write in " +
188 this.name.getMethodName());
189 final Configuration conf = HBaseConfiguration.create();
190
191 conf.setInt("hbase.hstore.flush.retries.number", 1);
192 User user = User.createUserForTesting(conf, this.name.getMethodName(),
193 new String[]{"foo"});
194
195 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
196 user.runAs(new PrivilegedExceptionAction<Object>() {
197 public Object run() throws Exception {
198
199 FileSystem fs = FileSystem.get(conf);
200 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
201 FaultyFileSystem ffs = (FaultyFileSystem)fs;
202
203
204 init(name.getMethodName(), conf);
205
206 long size = store.memstore.getFlushableSize();
207 Assert.assertEquals(0, size);
208 LOG.info("Adding some data");
209 long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
210 size = store.memstore.getFlushableSize();
211 Assert.assertEquals(kvSize, size);
212
213 try {
214 LOG.info("Flushing");
215 flushStore(store, id++);
216 Assert.fail("Didn't bubble up IOE!");
217 } catch (IOException ioe) {
218 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
219 }
220 size = store.memstore.getFlushableSize();
221 Assert.assertEquals(kvSize, size);
222 store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
223
224
225 Assert.assertEquals(kvSize, size);
226 ffs.fault.set(false);
227 flushStore(store, id++);
228 size = store.memstore.getFlushableSize();
229
230 Assert.assertEquals(kvSize, size);
231 flushStore(store, id++);
232 size = store.memstore.getFlushableSize();
233 Assert.assertEquals(0, size);
234 return null;
235 }
236 });
237 }
238
239
240
241
242
243 @Test
244 public void testCreateWriter() throws Exception {
245 Configuration conf = HBaseConfiguration.create();
246 FileSystem fs = FileSystem.get(conf);
247
248 HColumnDescriptor hcd = new HColumnDescriptor(family);
249 hcd.setCompressionType(Compression.Algorithm.GZ);
250 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
251 init(name.getMethodName(), conf, hcd);
252
253
254 StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
255 Path path = writer.getPath();
256 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
257 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
258 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
259 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
260 writer.close();
261
262
263 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
264 Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
265 Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
266 reader.close();
267 }
268
269 @Test
270 public void testDeleteExpiredStoreFiles() throws Exception {
271 int storeFileNum = 4;
272 int ttl = 4;
273 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
274 EnvironmentEdgeManagerTestHelper.injectEdge(edge);
275
276 Configuration conf = HBaseConfiguration.create();
277
278 conf.setBoolean("hbase.store.delete.expired.storefile", true);
279
280 conf.setInt(CompactionConfiguration.MIN_KEY, 5);
281
282 HColumnDescriptor hcd = new HColumnDescriptor(family);
283 hcd.setTimeToLive(ttl);
284 init(name.getMethodName(), conf, hcd);
285
286 long storeTtl = this.store.getScanInfo().getTtl();
287 long sleepTime = storeTtl / storeFileNum;
288 long timeStamp;
289
290
291 for (int i = 1; i <= storeFileNum; i++) {
292 LOG.info("Adding some data for the store file #" + i);
293 timeStamp = EnvironmentEdgeManager.currentTimeMillis();
294 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
295 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
296 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
297 flush(i);
298 edge.incrementTime(sleepTime);
299 }
300
301
302 Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
303
304
305
306 for (int i = 1; i <= storeFileNum - 1; i++) {
307
308 Assert.assertNull(this.store.requestCompaction());
309 Collection<StoreFile> sfs = this.store.getStorefiles();
310
311 Assert.assertEquals(storeFileNum - i, sfs.size());
312
313 for (StoreFile sf : sfs) {
314 Assert.assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTimeMillis() - storeTtl));
315 }
316
317 edge.incrementTime(sleepTime);
318 }
319 Assert.assertNull(this.store.requestCompaction());
320 Collection<StoreFile> sfs = this.store.getStorefiles();
321
322 Assert.assertEquals(1, sfs.size());
323 long ts = sfs.iterator().next().getReader().getMaxTimestamp();
324 Assert.assertTrue(ts < (edge.currentTimeMillis() - storeTtl));
325 }
326
327 @Test
328 public void testLowestModificationTime() throws Exception {
329 Configuration conf = HBaseConfiguration.create();
330 FileSystem fs = FileSystem.get(conf);
331
332 init(name.getMethodName(), conf);
333
334 int storeFileNum = 4;
335 for (int i = 1; i <= storeFileNum; i++) {
336 LOG.info("Adding some data for the store file #"+i);
337 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
338 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
339 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
340 flush(i);
341 }
342
343 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
344 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
345 Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
346
347
348 store.compact(store.requestCompaction());
349 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
350 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
351 Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
352 }
353
354 private static long getLowestTimeStampFromFS(FileSystem fs,
355 final Collection<StoreFile> candidates) throws IOException {
356 long minTs = Long.MAX_VALUE;
357 if (candidates.isEmpty()) {
358 return minTs;
359 }
360 Path[] p = new Path[candidates.size()];
361 int i = 0;
362 for (StoreFile sf : candidates) {
363 p[i] = sf.getPath();
364 ++i;
365 }
366
367 FileStatus[] stats = fs.listStatus(p);
368 if (stats == null || stats.length == 0) {
369 return minTs;
370 }
371 for (FileStatus s : stats) {
372 minTs = Math.min(minTs, s.getModificationTime());
373 }
374 return minTs;
375 }
376
377
378
379
380
381 private static final int BLOCKSIZE_SMALL = 8192;
382
383
384
385
386 @Test
387 public void testEmptyStoreFile() throws IOException {
388 init(this.name.getMethodName());
389
390 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
391 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
392 flush(1);
393
394
395 StoreFile f = this.store.getStorefiles().iterator().next();
396 Path storedir = f.getPath().getParent();
397 long seqid = f.getMaxSequenceId();
398 Configuration c = HBaseConfiguration.create();
399 FileSystem fs = FileSystem.get(c);
400 HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
401 StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
402 fs)
403 .withOutputDir(storedir)
404 .withFileContext(meta)
405 .build();
406 w.appendMetadata(seqid + 1, false);
407 w.close();
408 this.store.close();
409
410 this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
411 Assert.assertEquals(2, this.store.getStorefilesCount());
412
413 result = HBaseTestingUtility.getFromStoreFile(store,
414 get.getRow(),
415 qualifiers);
416 Assert.assertEquals(1, result.size());
417 }
418
419
420
421
422
423 @Test
424 public void testGet_FromMemStoreOnly() throws IOException {
425 init(this.name.getMethodName());
426
427
428 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
429 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
430 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
431 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
432 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
433 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
434
435
436 result = HBaseTestingUtility.getFromStoreFile(store,
437 get.getRow(), qualifiers);
438
439
440 assertCheck();
441 }
442
443
444
445
446
447 @Test
448 public void testGet_FromFilesOnly() throws IOException {
449 init(this.name.getMethodName());
450
451
452 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
453 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
454
455 flush(1);
456
457
458 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
459 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
460
461 flush(2);
462
463
464 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
465 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
466
467 flush(3);
468
469
470 result = HBaseTestingUtility.getFromStoreFile(store,
471 get.getRow(),
472 qualifiers);
473
474
475
476 Collections.sort(result, KeyValue.COMPARATOR);
477
478
479 assertCheck();
480 }
481
482
483
484
485
486 @Test
487 public void testGet_FromMemStoreAndFiles() throws IOException {
488 init(this.name.getMethodName());
489
490
491 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
492 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
493
494 flush(1);
495
496
497 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
498 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
499
500 flush(2);
501
502
503 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
504 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
505
506
507 result = HBaseTestingUtility.getFromStoreFile(store,
508 get.getRow(), qualifiers);
509
510
511 Collections.sort(result, KeyValue.COMPARATOR);
512
513
514 assertCheck();
515 }
516
517 private void flush(int storeFilessize) throws IOException{
518 this.store.snapshot();
519 flushStore(store, id++);
520 Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
521 Assert.assertEquals(0, this.store.memstore.kvset.size());
522 }
523
524 private void assertCheck() {
525 Assert.assertEquals(expected.size(), result.size());
526 for(int i=0; i<expected.size(); i++) {
527 Assert.assertEquals(expected.get(i), result.get(i));
528 }
529 }
530
531
532
533
534
535
536
537 @Test
538 public void testIncrementColumnValue_ICVDuringFlush()
539 throws IOException, InterruptedException {
540 init(this.name.getMethodName());
541
542 long oldValue = 1L;
543 long newValue = 3L;
544 this.store.add(new KeyValue(row, family, qf1,
545 System.currentTimeMillis(),
546 Bytes.toBytes(oldValue)));
547
548
549 this.store.snapshot();
550
551
552 this.store.add(new KeyValue(row, family, qf2,
553 System.currentTimeMillis(),
554 Bytes.toBytes(oldValue)));
555
556
557 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
558
559
560 Assert.assertTrue(ret > 0);
561
562
563 flushStore(store, id++);
564 Assert.assertEquals(1, this.store.getStorefiles().size());
565
566 Assert.assertEquals(2, this.store.memstore.kvset.size());
567
568
569 Get get = new Get(row);
570 get.addColumn(family, qf1);
571 get.setMaxVersions();
572 List<Cell> results = new ArrayList<Cell>();
573
574 results = HBaseTestingUtility.getFromStoreFile(store, get);
575 Assert.assertEquals(2, results.size());
576
577 long ts1 = results.get(0).getTimestamp();
578 long ts2 = results.get(1).getTimestamp();
579
580 Assert.assertTrue(ts1 > ts2);
581
582 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
583 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
584 }
585
586 @After
587 public void tearDown() throws Exception {
588 EnvironmentEdgeManagerTestHelper.reset();
589 }
590
591 @Test
592 public void testICV_negMemstoreSize() throws IOException {
593 init(this.name.getMethodName());
594
595 long time = 100;
596 ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
597 ee.setValue(time);
598 EnvironmentEdgeManagerTestHelper.injectEdge(ee);
599 long newValue = 3L;
600 long size = 0;
601
602
603 size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
604 System.currentTimeMillis(),
605 Bytes.toBytes(newValue)));
606 size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
607 System.currentTimeMillis(),
608 Bytes.toBytes(newValue)));
609 size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
610 System.currentTimeMillis(),
611 Bytes.toBytes(newValue)));
612 size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
613 System.currentTimeMillis(),
614 Bytes.toBytes(newValue)));
615 size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
616 System.currentTimeMillis(),
617 Bytes.toBytes(newValue)));
618
619
620 for ( int i = 0 ; i < 10000 ; ++i) {
621 newValue++;
622
623 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
624 long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
625
626 if (ret != 0) System.out.println("ret: " + ret);
627 if (ret2 != 0) System.out.println("ret2: " + ret2);
628
629 Assert.assertTrue("ret: " + ret, ret >= 0);
630 size += ret;
631 Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
632 size += ret2;
633
634
635 if (i % 1000 == 0)
636 ee.setValue(++time);
637 }
638
639 long computedSize=0;
640 for (KeyValue kv : this.store.memstore.kvset) {
641 long kvsize = MemStore.heapSizeChange(kv, true);
642
643 computedSize += kvsize;
644 }
645 Assert.assertEquals(computedSize, size);
646 }
647
648 @Test
649 public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
650 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
651 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
652 init(this.name.getMethodName());
653
654 long oldValue = 1L;
655 long newValue = 3L;
656 this.store.add(new KeyValue(row, family, qf1,
657 EnvironmentEdgeManager.currentTimeMillis(),
658 Bytes.toBytes(oldValue)));
659
660
661 this.store.snapshot();
662
663
664 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
665
666
667 Assert.assertTrue(ret > 0);
668
669
670 flushStore(store, id++);
671 Assert.assertEquals(1, this.store.getStorefiles().size());
672 Assert.assertEquals(1, this.store.memstore.kvset.size());
673
674
675 newValue += 1;
676 this.store.updateColumnValue(row, family, qf1, newValue);
677
678
679 newValue += 1;
680 this.store.updateColumnValue(row, family, qf1, newValue);
681
682
683
684
685
686 Get get = new Get(row);
687 get.addColumn(family, qf1);
688 get.setMaxVersions();
689 List<Cell> results = new ArrayList<Cell>();
690
691 results = HBaseTestingUtility.getFromStoreFile(store, get);
692 Assert.assertEquals(2, results.size());
693
694 long ts1 = results.get(0).getTimestamp();
695 long ts2 = results.get(1).getTimestamp();
696
697 Assert.assertTrue(ts1 > ts2);
698 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
699 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
700
701 mee.setValue(2);
702 newValue += 1;
703 this.store.updateColumnValue(row, family, qf1, newValue);
704
705 results = HBaseTestingUtility.getFromStoreFile(store, get);
706 Assert.assertEquals(2, results.size());
707
708 ts1 = results.get(0).getTimestamp();
709 ts2 = results.get(1).getTimestamp();
710
711 Assert.assertTrue(ts1 > ts2);
712 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
713 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
714 }
715
716 @Test
717 public void testHandleErrorsInFlush() throws Exception {
718 LOG.info("Setting up a faulty file system that cannot write");
719
720 final Configuration conf = HBaseConfiguration.create();
721 User user = User.createUserForTesting(conf,
722 "testhandleerrorsinflush", new String[]{"foo"});
723
724 conf.setClass("fs.file.impl", FaultyFileSystem.class,
725 FileSystem.class);
726 user.runAs(new PrivilegedExceptionAction<Object>() {
727 public Object run() throws Exception {
728
729 FileSystem fs = FileSystem.get(conf);
730 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
731
732
733 init(name.getMethodName(), conf);
734
735 LOG.info("Adding some data");
736 store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
737 store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
738 store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
739
740 LOG.info("Before flush, we should have no files");
741
742 Collection<StoreFileInfo> files =
743 store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
744 Assert.assertEquals(0, files != null ? files.size() : 0);
745
746
747 try {
748 LOG.info("Flushing");
749 flush(1);
750 Assert.fail("Didn't bubble up IOE!");
751 } catch (IOException ioe) {
752 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
753 }
754
755 LOG.info("After failed flush, we should still have no files!");
756 files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
757 Assert.assertEquals(0, files != null ? files.size() : 0);
758 store.getHRegion().getLog().closeAndDelete();
759 return null;
760 }
761 });
762 FileSystem.closeAllForUGI(user.getUGI());
763 }
764
765
766
767
768
769 static class FaultyFileSystem extends FilterFileSystem {
770 List<SoftReference<FaultyOutputStream>> outStreams =
771 new ArrayList<SoftReference<FaultyOutputStream>>();
772 private long faultPos = 200;
773 AtomicBoolean fault = new AtomicBoolean(true);
774
775 public FaultyFileSystem() {
776 super(new LocalFileSystem());
777 System.err.println("Creating faulty!");
778 }
779
780 @Override
781 public FSDataOutputStream create(Path p) throws IOException {
782 return new FaultyOutputStream(super.create(p), faultPos, fault);
783 }
784
785 @Override
786 public FSDataOutputStream create(Path f, FsPermission permission,
787 boolean overwrite, int bufferSize, short replication, long blockSize,
788 Progressable progress) throws IOException {
789 return new FaultyOutputStream(super.create(f, permission,
790 overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
791 }
792
793 public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
794 int bufferSize, short replication, long blockSize, Progressable progress)
795 throws IOException {
796
797
798 return create(f, overwrite, bufferSize, replication, blockSize, progress);
799 }
800 }
801
802 static class FaultyOutputStream extends FSDataOutputStream {
803 volatile long faultPos = Long.MAX_VALUE;
804 private final AtomicBoolean fault;
805
806 public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
807 throws IOException {
808 super(out, null);
809 this.faultPos = faultPos;
810 this.fault = fault;
811 }
812
813 @Override
814 public void write(byte[] buf, int offset, int length) throws IOException {
815 System.err.println("faulty stream write at pos " + getPos());
816 injectFault();
817 super.write(buf, offset, length);
818 }
819
820 private void injectFault() throws IOException {
821 if (this.fault.get() && getPos() >= faultPos) {
822 throw new IOException("Fault injected");
823 }
824 }
825 }
826
827 private static void flushStore(HStore store, long id) throws IOException {
828 StoreFlushContext storeFlushCtx = store.createFlushContext(id);
829 storeFlushCtx.prepare();
830 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
831 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
832 }
833
834
835
836
837
838
839
840
841
842 List<Cell> getKeyValueSet(long[] timestamps, int numRows,
843 byte[] qualifier, byte[] family) {
844 List<Cell> kvList = new ArrayList<Cell>();
845 for (int i=1;i<=numRows;i++) {
846 byte[] b = Bytes.toBytes(i);
847 for (long timestamp: timestamps) {
848 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
849 }
850 }
851 return kvList;
852 }
853
854
855
856
857
858 @Test
859 public void testMultipleTimestamps() throws IOException {
860 int numRows = 1;
861 long[] timestamps1 = new long[] {1,5,10,20};
862 long[] timestamps2 = new long[] {30,80};
863
864 init(this.name.getMethodName());
865
866 List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
867 for (Cell kv : kvList1) {
868 this.store.add(KeyValueUtil.ensureKeyValue(kv));
869 }
870
871 this.store.snapshot();
872 flushStore(store, id++);
873
874 List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
875 for(Cell kv : kvList2) {
876 this.store.add(KeyValueUtil.ensureKeyValue(kv));
877 }
878
879 List<Cell> result;
880 Get get = new Get(Bytes.toBytes(1));
881 get.addColumn(family,qf1);
882
883 get.setTimeRange(0,15);
884 result = HBaseTestingUtility.getFromStoreFile(store, get);
885 Assert.assertTrue(result.size()>0);
886
887 get.setTimeRange(40,90);
888 result = HBaseTestingUtility.getFromStoreFile(store, get);
889 Assert.assertTrue(result.size()>0);
890
891 get.setTimeRange(10,45);
892 result = HBaseTestingUtility.getFromStoreFile(store, get);
893 Assert.assertTrue(result.size()>0);
894
895 get.setTimeRange(80,145);
896 result = HBaseTestingUtility.getFromStoreFile(store, get);
897 Assert.assertTrue(result.size()>0);
898
899 get.setTimeRange(1,2);
900 result = HBaseTestingUtility.getFromStoreFile(store, get);
901 Assert.assertTrue(result.size()>0);
902
903 get.setTimeRange(90,200);
904 result = HBaseTestingUtility.getFromStoreFile(store, get);
905 Assert.assertTrue(result.size()==0);
906 }
907
908
909
910
911
912
913 @Test
914 public void testSplitWithEmptyColFam() throws IOException {
915 init(this.name.getMethodName());
916 Assert.assertNull(store.getSplitPoint());
917 store.getHRegion().forceSplit(null);
918 Assert.assertNull(store.getSplitPoint());
919 store.getHRegion().clearSplit();
920 }
921
922 @Test
923 public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
924 final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
925 long anyValue = 10;
926
927
928
929
930 Configuration conf = HBaseConfiguration.create();
931 conf.setLong(CONFIG_KEY, anyValue);
932 init(name.getMethodName() + "-xml", conf);
933 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
934 Assert.assertFalse(store.throttleCompaction(anyValue));
935
936
937 --anyValue;
938 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
939 HColumnDescriptor hcd = new HColumnDescriptor(family);
940 htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
941 init(name.getMethodName() + "-htd", conf, htd, hcd);
942 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
943 Assert.assertFalse(store.throttleCompaction(anyValue));
944
945
946 --anyValue;
947 hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
948 init(name.getMethodName() + "-hcd", conf, htd, hcd);
949 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
950 Assert.assertFalse(store.throttleCompaction(anyValue));
951 }
952
953 public static class DummyStoreEngine extends DefaultStoreEngine {
954 public static DefaultCompactor lastCreatedCompactor = null;
955 @Override
956 protected void createComponents(
957 Configuration conf, Store store, KVComparator comparator) throws IOException {
958 super.createComponents(conf, store, comparator);
959 lastCreatedCompactor = this.compactor;
960 }
961 }
962
963 @Test
964 public void testStoreUsesSearchEngineOverride() throws Exception {
965 Configuration conf = HBaseConfiguration.create();
966 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
967 init(this.name.getMethodName(), conf);
968 Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
969 this.store.storeEngine.getCompactor());
970 }
971 }