1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.lang.ref.SoftReference;
24 import java.security.PrivilegedExceptionAction;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.NavigableSet;
31 import java.util.concurrent.ConcurrentSkipListSet;
32 import java.util.concurrent.atomic.AtomicBoolean;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.FilterFileSystem;
41 import org.apache.hadoop.fs.LocalFileSystem;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.fs.permission.FsPermission;
44 import org.apache.hadoop.hbase.Cell;
45 import org.apache.hadoop.hbase.CellUtil;
46 import org.apache.hadoop.hbase.HBaseConfiguration;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HColumnDescriptor;
49 import org.apache.hadoop.hbase.HRegionInfo;
50 import org.apache.hadoop.hbase.HTableDescriptor;
51 import org.apache.hadoop.hbase.KeyValue;
52 import org.apache.hadoop.hbase.KeyValue.KVComparator;
53 import org.apache.hadoop.hbase.KeyValueUtil;
54 import org.apache.hadoop.hbase.testclassification.MediumTests;
55 import org.apache.hadoop.hbase.TableName;
56 import org.apache.hadoop.hbase.client.Get;
57 import org.apache.hadoop.hbase.io.compress.Compression;
58 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
59 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
60 import org.apache.hadoop.hbase.io.hfile.HFile;
61 import org.apache.hadoop.hbase.io.hfile.HFileContext;
62 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
63 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
64 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
65 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
66 import org.apache.hadoop.hbase.regionserver.wal.HLog;
67 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
68 import org.apache.hadoop.hbase.security.User;
69 import org.apache.hadoop.hbase.util.Bytes;
70 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
71 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
72 import org.apache.hadoop.hbase.util.FSUtils;
73 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
74 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
75 import org.apache.hadoop.util.Progressable;
76 import org.junit.After;
77 import org.junit.Assert;
78 import org.junit.Before;
79 import org.junit.Rule;
80 import org.junit.Test;
81 import org.junit.experimental.categories.Category;
82 import org.junit.rules.TestName;
83 import org.mockito.Mockito;
84
85
86
87
88 @Category(MediumTests.class)
89 public class TestStore {
90 public static final Log LOG = LogFactory.getLog(TestStore.class);
91 @Rule public TestName name = new TestName();
92
93 HStore store;
94 byte [] table = Bytes.toBytes("table");
95 byte [] family = Bytes.toBytes("family");
96
97 byte [] row = Bytes.toBytes("row");
98 byte [] row2 = Bytes.toBytes("row2");
99 byte [] qf1 = Bytes.toBytes("qf1");
100 byte [] qf2 = Bytes.toBytes("qf2");
101 byte [] qf3 = Bytes.toBytes("qf3");
102 byte [] qf4 = Bytes.toBytes("qf4");
103 byte [] qf5 = Bytes.toBytes("qf5");
104 byte [] qf6 = Bytes.toBytes("qf6");
105
106 NavigableSet<byte[]> qualifiers =
107 new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
108
109 List<Cell> expected = new ArrayList<Cell>();
110 List<Cell> result = new ArrayList<Cell>();
111
112 long id = System.currentTimeMillis();
113 Get get = new Get(row);
114
115 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
116 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
117
118
119
120
121
122
123 @Before
124 public void setUp() throws IOException {
125 qualifiers.add(qf1);
126 qualifiers.add(qf3);
127 qualifiers.add(qf5);
128
129 Iterator<byte[]> iter = qualifiers.iterator();
130 while(iter.hasNext()){
131 byte [] next = iter.next();
132 expected.add(new KeyValue(row, family, next, 1, (byte[])null));
133 get.addColumn(family, next);
134 }
135 }
136
137 private void init(String methodName) throws IOException {
138 init(methodName, HBaseConfiguration.create());
139 }
140
141 private void init(String methodName, Configuration conf)
142 throws IOException {
143 HColumnDescriptor hcd = new HColumnDescriptor(family);
144
145
146 hcd.setMaxVersions(4);
147 init(methodName, conf, hcd);
148 }
149
150 private void init(String methodName, Configuration conf,
151 HColumnDescriptor hcd) throws IOException {
152 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
153 init(methodName, conf, htd, hcd);
154 }
155
156 @SuppressWarnings("deprecation")
157 private Store init(String methodName, Configuration conf, HTableDescriptor htd,
158 HColumnDescriptor hcd) throws IOException {
159
160 Path basedir = new Path(DIR+methodName);
161 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
162 String logName = "logs";
163 Path logdir = new Path(basedir, logName);
164
165 FileSystem fs = FileSystem.get(conf);
166
167 fs.delete(logdir, true);
168
169 htd.addFamily(hcd);
170 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
171 HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
172 HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
173
174 store = new HStore(region, hcd, conf);
175 return store;
176 }
177
178
179
180
181
182
183 @Test
184 public void testFlushSizeAccounting() throws Exception {
185 LOG.info("Setting up a faulty file system that cannot write in " +
186 this.name.getMethodName());
187 final Configuration conf = HBaseConfiguration.create();
188
189 conf.setInt("hbase.hstore.flush.retries.number", 1);
190 User user = User.createUserForTesting(conf, this.name.getMethodName(),
191 new String[]{"foo"});
192
193 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
194 user.runAs(new PrivilegedExceptionAction<Object>() {
195 public Object run() throws Exception {
196
197 FileSystem fs = FileSystem.get(conf);
198 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
199 FaultyFileSystem ffs = (FaultyFileSystem)fs;
200
201
202 init(name.getMethodName(), conf);
203
204 long size = store.memstore.getFlushableSize();
205 Assert.assertEquals(0, size);
206 LOG.info("Adding some data");
207 long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
208 size = store.memstore.getFlushableSize();
209 Assert.assertEquals(kvSize, size);
210
211 try {
212 LOG.info("Flushing");
213 flushStore(store, id++);
214 Assert.fail("Didn't bubble up IOE!");
215 } catch (IOException ioe) {
216 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
217 }
218 size = store.memstore.getFlushableSize();
219 Assert.assertEquals(kvSize, size);
220 store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
221
222
223 Assert.assertEquals(kvSize, size);
224 ffs.fault.set(false);
225 flushStore(store, id++);
226 size = store.memstore.getFlushableSize();
227
228 Assert.assertEquals(kvSize, size);
229 flushStore(store, id++);
230 size = store.memstore.getFlushableSize();
231 Assert.assertEquals(0, size);
232 return null;
233 }
234 });
235 }
236
237
238
239
240
241 @Test
242 public void testCreateWriter() throws Exception {
243 Configuration conf = HBaseConfiguration.create();
244 FileSystem fs = FileSystem.get(conf);
245
246 HColumnDescriptor hcd = new HColumnDescriptor(family);
247 hcd.setCompressionType(Compression.Algorithm.GZ);
248 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
249 init(name.getMethodName(), conf, hcd);
250
251
252 StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
253 Path path = writer.getPath();
254 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
255 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
256 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
257 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
258 writer.close();
259
260
261 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
262 Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
263 Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
264 reader.close();
265 }
266
267 @Test
268 public void testDeleteExpiredStoreFiles() throws Exception {
269 testDeleteExpiredStoreFiles(0);
270 testDeleteExpiredStoreFiles(1);
271 }
272
273
274
275
276 public void testDeleteExpiredStoreFiles(int minVersions) throws Exception {
277 int storeFileNum = 4;
278 int ttl = 4;
279 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
280 EnvironmentEdgeManagerTestHelper.injectEdge(edge);
281
282 Configuration conf = HBaseConfiguration.create();
283
284 conf.setBoolean("hbase.store.delete.expired.storefile", true);
285
286 conf.setInt(CompactionConfiguration.MIN_KEY, 5);
287
288 HColumnDescriptor hcd = new HColumnDescriptor(family);
289 hcd.setMinVersions(minVersions);
290 hcd.setTimeToLive(ttl);
291 init(name.getMethodName(), conf, hcd);
292
293 long storeTtl = this.store.getScanInfo().getTtl();
294 long sleepTime = storeTtl / storeFileNum;
295 long timeStamp;
296
297
298 for (int i = 1; i <= storeFileNum; i++) {
299 LOG.info("Adding some data for the store file #" + i);
300 timeStamp = EnvironmentEdgeManager.currentTimeMillis();
301 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
302 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
303 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
304 flush(i);
305 edge.incrementTime(sleepTime);
306 }
307
308
309 Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
310
311
312
313 for (int i = 1; i <= storeFileNum - 1; i++) {
314
315 Assert.assertNull(this.store.requestCompaction());
316 Collection<StoreFile> sfs = this.store.getStorefiles();
317
318 if (minVersions == 0) {
319 Assert.assertEquals(storeFileNum - i, sfs.size());
320
321 for (StoreFile sf : sfs) {
322 Assert.assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTimeMillis() - storeTtl));
323 }
324 } else {
325 Assert.assertEquals(storeFileNum, sfs.size());
326 }
327
328 edge.incrementTime(sleepTime);
329 }
330 Assert.assertNull(this.store.requestCompaction());
331 Collection<StoreFile> sfs = this.store.getStorefiles();
332
333 if (minVersions == 0) {
334 Assert.assertEquals(1, sfs.size());
335 }
336 long ts = sfs.iterator().next().getReader().getMaxTimestamp();
337 Assert.assertTrue(ts < (edge.currentTimeMillis() - storeTtl));
338 }
339
340 @Test
341 public void testLowestModificationTime() throws Exception {
342 Configuration conf = HBaseConfiguration.create();
343 FileSystem fs = FileSystem.get(conf);
344
345 init(name.getMethodName(), conf);
346
347 int storeFileNum = 4;
348 for (int i = 1; i <= storeFileNum; i++) {
349 LOG.info("Adding some data for the store file #"+i);
350 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
351 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
352 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
353 flush(i);
354 }
355
356 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
357 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
358 Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
359
360
361 store.compact(store.requestCompaction());
362 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
363 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
364 Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
365 }
366
367 private static long getLowestTimeStampFromFS(FileSystem fs,
368 final Collection<StoreFile> candidates) throws IOException {
369 long minTs = Long.MAX_VALUE;
370 if (candidates.isEmpty()) {
371 return minTs;
372 }
373 Path[] p = new Path[candidates.size()];
374 int i = 0;
375 for (StoreFile sf : candidates) {
376 p[i] = sf.getPath();
377 ++i;
378 }
379
380 FileStatus[] stats = fs.listStatus(p);
381 if (stats == null || stats.length == 0) {
382 return minTs;
383 }
384 for (FileStatus s : stats) {
385 minTs = Math.min(minTs, s.getModificationTime());
386 }
387 return minTs;
388 }
389
390
391
392
393
394 private static final int BLOCKSIZE_SMALL = 8192;
395
396
397
398
399 @Test
400 public void testEmptyStoreFile() throws IOException {
401 init(this.name.getMethodName());
402
403 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
404 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
405 flush(1);
406
407
408 StoreFile f = this.store.getStorefiles().iterator().next();
409 Path storedir = f.getPath().getParent();
410 long seqid = f.getMaxSequenceId();
411 Configuration c = HBaseConfiguration.create();
412 FileSystem fs = FileSystem.get(c);
413 HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
414 StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
415 fs)
416 .withOutputDir(storedir)
417 .withFileContext(meta)
418 .build();
419 w.appendMetadata(seqid + 1, false);
420 w.close();
421 this.store.close();
422
423 this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
424 Assert.assertEquals(2, this.store.getStorefilesCount());
425
426 result = HBaseTestingUtility.getFromStoreFile(store,
427 get.getRow(),
428 qualifiers);
429 Assert.assertEquals(1, result.size());
430 }
431
432
433
434
435
436 @Test
437 public void testGet_FromMemStoreOnly() throws IOException {
438 init(this.name.getMethodName());
439
440
441 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
442 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
443 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
444 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
445 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
446 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
447
448
449 result = HBaseTestingUtility.getFromStoreFile(store,
450 get.getRow(), qualifiers);
451
452
453 assertCheck();
454 }
455
456
457
458
459
460 @Test
461 public void testGet_FromFilesOnly() throws IOException {
462 init(this.name.getMethodName());
463
464
465 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
466 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
467
468 flush(1);
469
470
471 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
472 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
473
474 flush(2);
475
476
477 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
478 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
479
480 flush(3);
481
482
483 result = HBaseTestingUtility.getFromStoreFile(store,
484 get.getRow(),
485 qualifiers);
486
487
488
489 Collections.sort(result, KeyValue.COMPARATOR);
490
491
492 assertCheck();
493 }
494
495
496
497
498
499 @Test
500 public void testGet_FromMemStoreAndFiles() throws IOException {
501 init(this.name.getMethodName());
502
503
504 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
505 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
506
507 flush(1);
508
509
510 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
511 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
512
513 flush(2);
514
515
516 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
517 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
518
519
520 result = HBaseTestingUtility.getFromStoreFile(store,
521 get.getRow(), qualifiers);
522
523
524 Collections.sort(result, KeyValue.COMPARATOR);
525
526
527 assertCheck();
528 }
529
530 private void flush(int storeFilessize) throws IOException{
531 this.store.snapshot();
532 flushStore(store, id++);
533 Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
534 Assert.assertEquals(0, this.store.memstore.kvset.size());
535 }
536
537 private void assertCheck() {
538 Assert.assertEquals(expected.size(), result.size());
539 for(int i=0; i<expected.size(); i++) {
540 Assert.assertEquals(expected.get(i), result.get(i));
541 }
542 }
543
544
545
546
547
548
549
550 @Test
551 public void testIncrementColumnValue_ICVDuringFlush()
552 throws IOException, InterruptedException {
553 init(this.name.getMethodName());
554
555 long oldValue = 1L;
556 long newValue = 3L;
557 this.store.add(new KeyValue(row, family, qf1,
558 System.currentTimeMillis(),
559 Bytes.toBytes(oldValue)));
560
561
562 this.store.snapshot();
563
564
565 this.store.add(new KeyValue(row, family, qf2,
566 System.currentTimeMillis(),
567 Bytes.toBytes(oldValue)));
568
569
570 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
571
572
573 Assert.assertTrue(ret > 0);
574
575
576 flushStore(store, id++);
577 Assert.assertEquals(1, this.store.getStorefiles().size());
578
579 Assert.assertEquals(2, this.store.memstore.kvset.size());
580
581
582 Get get = new Get(row);
583 get.addColumn(family, qf1);
584 get.setMaxVersions();
585 List<Cell> results = new ArrayList<Cell>();
586
587 results = HBaseTestingUtility.getFromStoreFile(store, get);
588 Assert.assertEquals(2, results.size());
589
590 long ts1 = results.get(0).getTimestamp();
591 long ts2 = results.get(1).getTimestamp();
592
593 Assert.assertTrue(ts1 > ts2);
594
595 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
596 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
597 }
598
599 @After
600 public void tearDown() throws Exception {
601 EnvironmentEdgeManagerTestHelper.reset();
602 }
603
604 @Test
605 public void testICV_negMemstoreSize() throws IOException {
606 init(this.name.getMethodName());
607
608 long time = 100;
609 ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
610 ee.setValue(time);
611 EnvironmentEdgeManagerTestHelper.injectEdge(ee);
612 long newValue = 3L;
613 long size = 0;
614
615
616 size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
617 System.currentTimeMillis(),
618 Bytes.toBytes(newValue)));
619 size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
620 System.currentTimeMillis(),
621 Bytes.toBytes(newValue)));
622 size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
623 System.currentTimeMillis(),
624 Bytes.toBytes(newValue)));
625 size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
626 System.currentTimeMillis(),
627 Bytes.toBytes(newValue)));
628 size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
629 System.currentTimeMillis(),
630 Bytes.toBytes(newValue)));
631
632
633 for ( int i = 0 ; i < 10000 ; ++i) {
634 newValue++;
635
636 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
637 long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
638
639 if (ret != 0) System.out.println("ret: " + ret);
640 if (ret2 != 0) System.out.println("ret2: " + ret2);
641
642 Assert.assertTrue("ret: " + ret, ret >= 0);
643 size += ret;
644 Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
645 size += ret2;
646
647
648 if (i % 1000 == 0)
649 ee.setValue(++time);
650 }
651
652 long computedSize=0;
653 for (KeyValue kv : this.store.memstore.kvset) {
654 long kvsize = MemStore.heapSizeChange(kv, true);
655
656 computedSize += kvsize;
657 }
658 Assert.assertEquals(computedSize, size);
659 }
660
661 @Test
662 public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
663 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
664 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
665 init(this.name.getMethodName());
666
667 long oldValue = 1L;
668 long newValue = 3L;
669 this.store.add(new KeyValue(row, family, qf1,
670 EnvironmentEdgeManager.currentTimeMillis(),
671 Bytes.toBytes(oldValue)));
672
673
674 this.store.snapshot();
675
676
677 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
678
679
680 Assert.assertTrue(ret > 0);
681
682
683 flushStore(store, id++);
684 Assert.assertEquals(1, this.store.getStorefiles().size());
685 Assert.assertEquals(1, this.store.memstore.kvset.size());
686
687
688 newValue += 1;
689 this.store.updateColumnValue(row, family, qf1, newValue);
690
691
692 newValue += 1;
693 this.store.updateColumnValue(row, family, qf1, newValue);
694
695
696
697
698
699 Get get = new Get(row);
700 get.addColumn(family, qf1);
701 get.setMaxVersions();
702 List<Cell> results = new ArrayList<Cell>();
703
704 results = HBaseTestingUtility.getFromStoreFile(store, get);
705 Assert.assertEquals(2, results.size());
706
707 long ts1 = results.get(0).getTimestamp();
708 long ts2 = results.get(1).getTimestamp();
709
710 Assert.assertTrue(ts1 > ts2);
711 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
712 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
713
714 mee.setValue(2);
715 newValue += 1;
716 this.store.updateColumnValue(row, family, qf1, newValue);
717
718 results = HBaseTestingUtility.getFromStoreFile(store, get);
719 Assert.assertEquals(2, results.size());
720
721 ts1 = results.get(0).getTimestamp();
722 ts2 = results.get(1).getTimestamp();
723
724 Assert.assertTrue(ts1 > ts2);
725 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
726 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
727 }
728
729 @Test
730 public void testHandleErrorsInFlush() throws Exception {
731 LOG.info("Setting up a faulty file system that cannot write");
732
733 final Configuration conf = HBaseConfiguration.create();
734 User user = User.createUserForTesting(conf,
735 "testhandleerrorsinflush", new String[]{"foo"});
736
737 conf.setClass("fs.file.impl", FaultyFileSystem.class,
738 FileSystem.class);
739 user.runAs(new PrivilegedExceptionAction<Object>() {
740 public Object run() throws Exception {
741
742 FileSystem fs = FileSystem.get(conf);
743 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
744
745
746 init(name.getMethodName(), conf);
747
748 LOG.info("Adding some data");
749 store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
750 store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
751 store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
752
753 LOG.info("Before flush, we should have no files");
754
755 Collection<StoreFileInfo> files =
756 store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
757 Assert.assertEquals(0, files != null ? files.size() : 0);
758
759
760 try {
761 LOG.info("Flushing");
762 flush(1);
763 Assert.fail("Didn't bubble up IOE!");
764 } catch (IOException ioe) {
765 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
766 }
767
768 LOG.info("After failed flush, we should still have no files!");
769 files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
770 Assert.assertEquals(0, files != null ? files.size() : 0);
771 store.getHRegion().getLog().closeAndDelete();
772 return null;
773 }
774 });
775 FileSystem.closeAllForUGI(user.getUGI());
776 }
777
778
779
780
781
782 static class FaultyFileSystem extends FilterFileSystem {
783 List<SoftReference<FaultyOutputStream>> outStreams =
784 new ArrayList<SoftReference<FaultyOutputStream>>();
785 private long faultPos = 200;
786 AtomicBoolean fault = new AtomicBoolean(true);
787
788 public FaultyFileSystem() {
789 super(new LocalFileSystem());
790 System.err.println("Creating faulty!");
791 }
792
793 @Override
794 public FSDataOutputStream create(Path p) throws IOException {
795 return new FaultyOutputStream(super.create(p), faultPos, fault);
796 }
797
798 @Override
799 public FSDataOutputStream create(Path f, FsPermission permission,
800 boolean overwrite, int bufferSize, short replication, long blockSize,
801 Progressable progress) throws IOException {
802 return new FaultyOutputStream(super.create(f, permission,
803 overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
804 }
805
806 public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
807 int bufferSize, short replication, long blockSize, Progressable progress)
808 throws IOException {
809
810
811 return create(f, overwrite, bufferSize, replication, blockSize, progress);
812 }
813 }
814
815 static class FaultyOutputStream extends FSDataOutputStream {
816 volatile long faultPos = Long.MAX_VALUE;
817 private final AtomicBoolean fault;
818
819 public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
820 throws IOException {
821 super(out, null);
822 this.faultPos = faultPos;
823 this.fault = fault;
824 }
825
826 @Override
827 public void write(byte[] buf, int offset, int length) throws IOException {
828 System.err.println("faulty stream write at pos " + getPos());
829 injectFault();
830 super.write(buf, offset, length);
831 }
832
833 private void injectFault() throws IOException {
834 if (this.fault.get() && getPos() >= faultPos) {
835 throw new IOException("Fault injected");
836 }
837 }
838 }
839
840 private static void flushStore(HStore store, long id) throws IOException {
841 StoreFlushContext storeFlushCtx = store.createFlushContext(id);
842 storeFlushCtx.prepare();
843 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
844 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
845 }
846
847
848
849
850
851
852
853
854
855 List<Cell> getKeyValueSet(long[] timestamps, int numRows,
856 byte[] qualifier, byte[] family) {
857 List<Cell> kvList = new ArrayList<Cell>();
858 for (int i=1;i<=numRows;i++) {
859 byte[] b = Bytes.toBytes(i);
860 for (long timestamp: timestamps) {
861 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
862 }
863 }
864 return kvList;
865 }
866
867
868
869
870
871 @Test
872 public void testMultipleTimestamps() throws IOException {
873 int numRows = 1;
874 long[] timestamps1 = new long[] {1,5,10,20};
875 long[] timestamps2 = new long[] {30,80};
876
877 init(this.name.getMethodName());
878
879 List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
880 for (Cell kv : kvList1) {
881 this.store.add(KeyValueUtil.ensureKeyValue(kv));
882 }
883
884 this.store.snapshot();
885 flushStore(store, id++);
886
887 List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
888 for(Cell kv : kvList2) {
889 this.store.add(KeyValueUtil.ensureKeyValue(kv));
890 }
891
892 List<Cell> result;
893 Get get = new Get(Bytes.toBytes(1));
894 get.addColumn(family,qf1);
895
896 get.setTimeRange(0,15);
897 result = HBaseTestingUtility.getFromStoreFile(store, get);
898 Assert.assertTrue(result.size()>0);
899
900 get.setTimeRange(40,90);
901 result = HBaseTestingUtility.getFromStoreFile(store, get);
902 Assert.assertTrue(result.size()>0);
903
904 get.setTimeRange(10,45);
905 result = HBaseTestingUtility.getFromStoreFile(store, get);
906 Assert.assertTrue(result.size()>0);
907
908 get.setTimeRange(80,145);
909 result = HBaseTestingUtility.getFromStoreFile(store, get);
910 Assert.assertTrue(result.size()>0);
911
912 get.setTimeRange(1,2);
913 result = HBaseTestingUtility.getFromStoreFile(store, get);
914 Assert.assertTrue(result.size()>0);
915
916 get.setTimeRange(90,200);
917 result = HBaseTestingUtility.getFromStoreFile(store, get);
918 Assert.assertTrue(result.size()==0);
919 }
920
921
922
923
924
925
926 @Test
927 public void testSplitWithEmptyColFam() throws IOException {
928 init(this.name.getMethodName());
929 Assert.assertNull(store.getSplitPoint());
930 store.getHRegion().forceSplit(null);
931 Assert.assertNull(store.getSplitPoint());
932 store.getHRegion().clearSplit();
933 }
934
935 @Test
936 public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
937 final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
938 long anyValue = 10;
939
940
941
942
943 Configuration conf = HBaseConfiguration.create();
944 conf.setLong(CONFIG_KEY, anyValue);
945 init(name.getMethodName() + "-xml", conf);
946 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
947 Assert.assertFalse(store.throttleCompaction(anyValue));
948
949
950 --anyValue;
951 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
952 HColumnDescriptor hcd = new HColumnDescriptor(family);
953 htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
954 init(name.getMethodName() + "-htd", conf, htd, hcd);
955 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
956 Assert.assertFalse(store.throttleCompaction(anyValue));
957
958
959 --anyValue;
960 hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
961 init(name.getMethodName() + "-hcd", conf, htd, hcd);
962 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
963 Assert.assertFalse(store.throttleCompaction(anyValue));
964 }
965
966 public static class DummyStoreEngine extends DefaultStoreEngine {
967 public static DefaultCompactor lastCreatedCompactor = null;
968 @Override
969 protected void createComponents(
970 Configuration conf, Store store, KVComparator comparator) throws IOException {
971 super.createComponents(conf, store, comparator);
972 lastCreatedCompactor = this.compactor;
973 }
974 }
975
976 @Test
977 public void testStoreUsesSearchEngineOverride() throws Exception {
978 Configuration conf = HBaseConfiguration.create();
979 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
980 init(this.name.getMethodName(), conf);
981 Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
982 this.store.storeEngine.getCompactor());
983 }
984 }