1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.lang.ref.SoftReference;
24 import java.security.PrivilegedExceptionAction;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.NavigableSet;
31 import java.util.concurrent.ConcurrentSkipListSet;
32 import java.util.concurrent.atomic.AtomicBoolean;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.FilterFileSystem;
41 import org.apache.hadoop.fs.LocalFileSystem;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.fs.permission.FsPermission;
44 import org.apache.hadoop.hbase.Cell;
45 import org.apache.hadoop.hbase.CellUtil;
46 import org.apache.hadoop.hbase.HBaseConfiguration;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HColumnDescriptor;
49 import org.apache.hadoop.hbase.HRegionInfo;
50 import org.apache.hadoop.hbase.HTableDescriptor;
51 import org.apache.hadoop.hbase.KeyValue;
52 import org.apache.hadoop.hbase.KeyValue.KVComparator;
53 import org.apache.hadoop.hbase.KeyValueUtil;
54 import org.apache.hadoop.hbase.MediumTests;
55 import org.apache.hadoop.hbase.TableName;
56 import org.apache.hadoop.hbase.client.Get;
57 import org.apache.hadoop.hbase.io.compress.Compression;
58 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
59 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
60 import org.apache.hadoop.hbase.io.hfile.HFile;
61 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
62 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
63 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
64 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
65 import org.apache.hadoop.hbase.regionserver.wal.HLog;
66 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
67 import org.apache.hadoop.hbase.security.User;
68 import org.apache.hadoop.hbase.util.Bytes;
69 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
70 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
71 import org.apache.hadoop.hbase.util.FSUtils;
72 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
73 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
74 import org.apache.hadoop.util.Progressable;
75 import org.junit.After;
76 import org.junit.Assert;
77 import org.junit.Before;
78 import org.junit.Rule;
79 import org.junit.Test;
80 import org.junit.experimental.categories.Category;
81 import org.junit.rules.TestName;
82 import org.mockito.Mockito;
83
84
85
86
87 @Category(MediumTests.class)
88 public class TestStore {
89 public static final Log LOG = LogFactory.getLog(TestStore.class);
90 @Rule public TestName name = new TestName();
91
92 HStore store;
93 byte [] table = Bytes.toBytes("table");
94 byte [] family = Bytes.toBytes("family");
95
96 byte [] row = Bytes.toBytes("row");
97 byte [] row2 = Bytes.toBytes("row2");
98 byte [] qf1 = Bytes.toBytes("qf1");
99 byte [] qf2 = Bytes.toBytes("qf2");
100 byte [] qf3 = Bytes.toBytes("qf3");
101 byte [] qf4 = Bytes.toBytes("qf4");
102 byte [] qf5 = Bytes.toBytes("qf5");
103 byte [] qf6 = Bytes.toBytes("qf6");
104
105 NavigableSet<byte[]> qualifiers =
106 new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
107
108 List<Cell> expected = new ArrayList<Cell>();
109 List<Cell> result = new ArrayList<Cell>();
110
111 long id = System.currentTimeMillis();
112 Get get = new Get(row);
113
114 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
115 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
116
117
118
119
120
121
122 @Before
123 public void setUp() throws IOException {
124 qualifiers.add(qf1);
125 qualifiers.add(qf3);
126 qualifiers.add(qf5);
127
128 Iterator<byte[]> iter = qualifiers.iterator();
129 while(iter.hasNext()){
130 byte [] next = iter.next();
131 expected.add(new KeyValue(row, family, next, 1, (byte[])null));
132 get.addColumn(family, next);
133 }
134 }
135
136 private void init(String methodName) throws IOException {
137 init(methodName, HBaseConfiguration.create());
138 }
139
140 private void init(String methodName, Configuration conf)
141 throws IOException {
142 HColumnDescriptor hcd = new HColumnDescriptor(family);
143
144
145 hcd.setMaxVersions(4);
146 init(methodName, conf, hcd);
147 }
148
149 private void init(String methodName, Configuration conf,
150 HColumnDescriptor hcd) throws IOException {
151 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
152 init(methodName, conf, htd, hcd);
153 }
154
155 @SuppressWarnings("deprecation")
156 private Store init(String methodName, Configuration conf, HTableDescriptor htd,
157 HColumnDescriptor hcd) throws IOException {
158
159 Path basedir = new Path(DIR+methodName);
160 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
161 String logName = "logs";
162 Path logdir = new Path(basedir, logName);
163
164 FileSystem fs = FileSystem.get(conf);
165
166 fs.delete(logdir, true);
167
168 htd.addFamily(hcd);
169 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
170 HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
171 HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
172
173 store = new HStore(region, hcd, conf);
174 return store;
175 }
176
177
178
179
180
181
182 @Test
183 public void testFlushSizeAccounting() throws Exception {
184 LOG.info("Setting up a faulty file system that cannot write in " +
185 this.name.getMethodName());
186 final Configuration conf = HBaseConfiguration.create();
187
188 conf.setInt("hbase.hstore.flush.retries.number", 1);
189 User user = User.createUserForTesting(conf, this.name.getMethodName(),
190 new String[]{"foo"});
191
192 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
193 user.runAs(new PrivilegedExceptionAction<Object>() {
194 public Object run() throws Exception {
195
196 FileSystem fs = FileSystem.get(conf);
197 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
198 FaultyFileSystem ffs = (FaultyFileSystem)fs;
199
200
201 init(name.getMethodName(), conf);
202
203 long size = store.memstore.getFlushableSize();
204 Assert.assertEquals(0, size);
205 LOG.info("Adding some data");
206 long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
207 size = store.memstore.getFlushableSize();
208 Assert.assertEquals(kvSize, size);
209
210 try {
211 LOG.info("Flushing");
212 flushStore(store, id++);
213 Assert.fail("Didn't bubble up IOE!");
214 } catch (IOException ioe) {
215 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
216 }
217 size = store.memstore.getFlushableSize();
218 Assert.assertEquals(kvSize, size);
219 store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
220
221
222 Assert.assertEquals(kvSize, size);
223 ffs.fault.set(false);
224 flushStore(store, id++);
225 size = store.memstore.getFlushableSize();
226
227 Assert.assertEquals(kvSize, size);
228 flushStore(store, id++);
229 size = store.memstore.getFlushableSize();
230 Assert.assertEquals(0, size);
231 return null;
232 }
233 });
234 }
235
236
237
238
239
240 @Test
241 public void testCreateWriter() throws Exception {
242 Configuration conf = HBaseConfiguration.create();
243 FileSystem fs = FileSystem.get(conf);
244
245 HColumnDescriptor hcd = new HColumnDescriptor(family);
246 hcd.setCompressionType(Compression.Algorithm.GZ);
247 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
248 init(name.getMethodName(), conf, hcd);
249
250
251 StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true);
252 Path path = writer.getPath();
253 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
254 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
255 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
256 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
257 writer.close();
258
259
260 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf));
261 Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
262 Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
263 reader.close();
264 }
265
266 @Test
267 public void testDeleteExpiredStoreFiles() throws Exception {
268 int storeFileNum = 4;
269 int ttl = 4;
270 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
271 EnvironmentEdgeManagerTestHelper.injectEdge(edge);
272
273 Configuration conf = HBaseConfiguration.create();
274
275 conf.setBoolean("hbase.store.delete.expired.storefile", true);
276 HColumnDescriptor hcd = new HColumnDescriptor(family);
277 hcd.setTimeToLive(ttl);
278 init(name.getMethodName(), conf, hcd);
279
280 long sleepTime = this.store.getScanInfo().getTtl() / storeFileNum;
281 long timeStamp;
282
283
284 for (int i = 1; i <= storeFileNum; i++) {
285 LOG.info("Adding some data for the store file #" + i);
286 timeStamp = EnvironmentEdgeManager.currentTimeMillis();
287 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
288 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
289 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
290 flush(i);
291 edge.incrementTime(sleepTime);
292 }
293
294
295 Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
296
297
298
299 for (int i = 1; i <= storeFileNum; i++) {
300
301 CompactionContext compaction = this.store.requestCompaction();
302 CompactionRequest cr = compaction.getRequest();
303
304
305 List<StoreFile> files = new ArrayList<StoreFile>(cr.getFiles());
306 Assert.assertEquals(Math.min(i, 2), cr.getFiles().size());
307 for (int j = 0; j < files.size(); j++) {
308 Assert.assertTrue(files.get(j).getReader().getMaxTimestamp() < (edge
309 .currentTimeMillis() - this.store.getScanInfo().getTtl()));
310 }
311
312
313 StoreFile compactedFile = this.store.compact(compaction).get(0);
314
315 Assert.assertEquals(0, compactedFile.getReader().getEntries());
316
317
318 edge.incrementTime(sleepTime);
319 }
320 }
321
322 @Test
323 public void testLowestModificationTime() throws Exception {
324 Configuration conf = HBaseConfiguration.create();
325 FileSystem fs = FileSystem.get(conf);
326
327 init(name.getMethodName(), conf);
328
329 int storeFileNum = 4;
330 for (int i = 1; i <= storeFileNum; i++) {
331 LOG.info("Adding some data for the store file #"+i);
332 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
333 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
334 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
335 flush(i);
336 }
337
338 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
339 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
340 Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
341
342
343 store.compact(store.requestCompaction());
344 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
345 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
346 Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
347 }
348
349 private static long getLowestTimeStampFromFS(FileSystem fs,
350 final Collection<StoreFile> candidates) throws IOException {
351 long minTs = Long.MAX_VALUE;
352 if (candidates.isEmpty()) {
353 return minTs;
354 }
355 Path[] p = new Path[candidates.size()];
356 int i = 0;
357 for (StoreFile sf : candidates) {
358 p[i] = sf.getPath();
359 ++i;
360 }
361
362 FileStatus[] stats = fs.listStatus(p);
363 if (stats == null || stats.length == 0) {
364 return minTs;
365 }
366 for (FileStatus s : stats) {
367 minTs = Math.min(minTs, s.getModificationTime());
368 }
369 return minTs;
370 }
371
372
373
374
375
376
377
378
379
380 @Test
381 public void testEmptyStoreFile() throws IOException {
382 init(this.name.getMethodName());
383
384 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
385 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
386 flush(1);
387
388
389 StoreFile f = this.store.getStorefiles().iterator().next();
390 Path storedir = f.getPath().getParent();
391 long seqid = f.getMaxSequenceId();
392 Configuration c = HBaseConfiguration.create();
393 FileSystem fs = FileSystem.get(c);
394 StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
395 fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL)
396 .withOutputDir(storedir)
397 .build();
398 w.appendMetadata(seqid + 1, false);
399 w.close();
400 this.store.close();
401
402 this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
403 Assert.assertEquals(2, this.store.getStorefilesCount());
404
405 result = HBaseTestingUtility.getFromStoreFile(store,
406 get.getRow(),
407 qualifiers);
408 Assert.assertEquals(1, result.size());
409 }
410
411
412
413
414
415 @Test
416 public void testGet_FromMemStoreOnly() throws IOException {
417 init(this.name.getMethodName());
418
419
420 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
421 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
422 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
423 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
424 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
425 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
426
427
428 result = HBaseTestingUtility.getFromStoreFile(store,
429 get.getRow(), qualifiers);
430
431
432 assertCheck();
433 }
434
435
436
437
438
439 @Test
440 public void testGet_FromFilesOnly() throws IOException {
441 init(this.name.getMethodName());
442
443
444 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
445 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
446
447 flush(1);
448
449
450 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
451 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
452
453 flush(2);
454
455
456 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
457 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
458
459 flush(3);
460
461
462 result = HBaseTestingUtility.getFromStoreFile(store,
463 get.getRow(),
464 qualifiers);
465
466
467
468 Collections.sort(result, KeyValue.COMPARATOR);
469
470
471 assertCheck();
472 }
473
474
475
476
477
478 @Test
479 public void testGet_FromMemStoreAndFiles() throws IOException {
480 init(this.name.getMethodName());
481
482
483 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
484 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
485
486 flush(1);
487
488
489 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
490 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
491
492 flush(2);
493
494
495 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
496 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
497
498
499 result = HBaseTestingUtility.getFromStoreFile(store,
500 get.getRow(), qualifiers);
501
502
503 Collections.sort(result, KeyValue.COMPARATOR);
504
505
506 assertCheck();
507 }
508
509 private void flush(int storeFilessize) throws IOException{
510 this.store.snapshot();
511 flushStore(store, id++);
512 Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
513 Assert.assertEquals(0, this.store.memstore.kvset.size());
514 }
515
516 private void assertCheck() {
517 Assert.assertEquals(expected.size(), result.size());
518 for(int i=0; i<expected.size(); i++) {
519 Assert.assertEquals(expected.get(i), result.get(i));
520 }
521 }
522
523
524
525
526
527
528
529 @Test
530 public void testIncrementColumnValue_ICVDuringFlush()
531 throws IOException, InterruptedException {
532 init(this.name.getMethodName());
533
534 long oldValue = 1L;
535 long newValue = 3L;
536 this.store.add(new KeyValue(row, family, qf1,
537 System.currentTimeMillis(),
538 Bytes.toBytes(oldValue)));
539
540
541 this.store.snapshot();
542
543
544 this.store.add(new KeyValue(row, family, qf2,
545 System.currentTimeMillis(),
546 Bytes.toBytes(oldValue)));
547
548
549 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
550
551
552 Assert.assertTrue(ret > 0);
553
554
555 flushStore(store, id++);
556 Assert.assertEquals(1, this.store.getStorefiles().size());
557
558 Assert.assertEquals(2, this.store.memstore.kvset.size());
559
560
561 Get get = new Get(row);
562 get.addColumn(family, qf1);
563 get.setMaxVersions();
564 List<Cell> results = new ArrayList<Cell>();
565
566 results = HBaseTestingUtility.getFromStoreFile(store, get);
567 Assert.assertEquals(2, results.size());
568
569 long ts1 = results.get(0).getTimestamp();
570 long ts2 = results.get(1).getTimestamp();
571
572 Assert.assertTrue(ts1 > ts2);
573
574 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
575 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
576 }
577
578 @After
579 public void tearDown() throws Exception {
580 EnvironmentEdgeManagerTestHelper.reset();
581 }
582
583 @Test
584 public void testICV_negMemstoreSize() throws IOException {
585 init(this.name.getMethodName());
586
587 long time = 100;
588 ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
589 ee.setValue(time);
590 EnvironmentEdgeManagerTestHelper.injectEdge(ee);
591 long newValue = 3L;
592 long size = 0;
593
594
595 size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
596 System.currentTimeMillis(),
597 Bytes.toBytes(newValue)));
598 size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
599 System.currentTimeMillis(),
600 Bytes.toBytes(newValue)));
601 size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
602 System.currentTimeMillis(),
603 Bytes.toBytes(newValue)));
604 size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
605 System.currentTimeMillis(),
606 Bytes.toBytes(newValue)));
607 size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
608 System.currentTimeMillis(),
609 Bytes.toBytes(newValue)));
610
611
612 for ( int i = 0 ; i < 10000 ; ++i) {
613 newValue++;
614
615 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
616 long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
617
618 if (ret != 0) System.out.println("ret: " + ret);
619 if (ret2 != 0) System.out.println("ret2: " + ret2);
620
621 Assert.assertTrue("ret: " + ret, ret >= 0);
622 size += ret;
623 Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
624 size += ret2;
625
626
627 if (i % 1000 == 0)
628 ee.setValue(++time);
629 }
630
631 long computedSize=0;
632 for (KeyValue kv : this.store.memstore.kvset) {
633 long kvsize = MemStore.heapSizeChange(kv, true);
634
635 computedSize += kvsize;
636 }
637 Assert.assertEquals(computedSize, size);
638 }
639
640 @Test
641 public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
642 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
643 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
644 init(this.name.getMethodName());
645
646 long oldValue = 1L;
647 long newValue = 3L;
648 this.store.add(new KeyValue(row, family, qf1,
649 EnvironmentEdgeManager.currentTimeMillis(),
650 Bytes.toBytes(oldValue)));
651
652
653 this.store.snapshot();
654
655
656 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
657
658
659 Assert.assertTrue(ret > 0);
660
661
662 flushStore(store, id++);
663 Assert.assertEquals(1, this.store.getStorefiles().size());
664 Assert.assertEquals(1, this.store.memstore.kvset.size());
665
666
667 newValue += 1;
668 this.store.updateColumnValue(row, family, qf1, newValue);
669
670
671 newValue += 1;
672 this.store.updateColumnValue(row, family, qf1, newValue);
673
674
675
676
677
678 Get get = new Get(row);
679 get.addColumn(family, qf1);
680 get.setMaxVersions();
681 List<Cell> results = new ArrayList<Cell>();
682
683 results = HBaseTestingUtility.getFromStoreFile(store, get);
684 Assert.assertEquals(2, results.size());
685
686 long ts1 = results.get(0).getTimestamp();
687 long ts2 = results.get(1).getTimestamp();
688
689 Assert.assertTrue(ts1 > ts2);
690 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
691 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
692
693 mee.setValue(2);
694 newValue += 1;
695 this.store.updateColumnValue(row, family, qf1, newValue);
696
697 results = HBaseTestingUtility.getFromStoreFile(store, get);
698 Assert.assertEquals(2, results.size());
699
700 ts1 = results.get(0).getTimestamp();
701 ts2 = results.get(1).getTimestamp();
702
703 Assert.assertTrue(ts1 > ts2);
704 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
705 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
706 }
707
708 @Test
709 public void testHandleErrorsInFlush() throws Exception {
710 LOG.info("Setting up a faulty file system that cannot write");
711
712 final Configuration conf = HBaseConfiguration.create();
713 User user = User.createUserForTesting(conf,
714 "testhandleerrorsinflush", new String[]{"foo"});
715
716 conf.setClass("fs.file.impl", FaultyFileSystem.class,
717 FileSystem.class);
718 user.runAs(new PrivilegedExceptionAction<Object>() {
719 public Object run() throws Exception {
720
721 FileSystem fs = FileSystem.get(conf);
722 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
723
724
725 init(name.getMethodName(), conf);
726
727 LOG.info("Adding some data");
728 store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
729 store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
730 store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
731
732 LOG.info("Before flush, we should have no files");
733
734 Collection<StoreFileInfo> files =
735 store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
736 Assert.assertEquals(0, files != null ? files.size() : 0);
737
738
739 try {
740 LOG.info("Flushing");
741 flush(1);
742 Assert.fail("Didn't bubble up IOE!");
743 } catch (IOException ioe) {
744 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
745 }
746
747 LOG.info("After failed flush, we should still have no files!");
748 files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
749 Assert.assertEquals(0, files != null ? files.size() : 0);
750 store.getHRegion().getLog().closeAndDelete();
751 return null;
752 }
753 });
754 FileSystem.closeAllForUGI(user.getUGI());
755 }
756
757
758
759
760
761 static class FaultyFileSystem extends FilterFileSystem {
762 List<SoftReference<FaultyOutputStream>> outStreams =
763 new ArrayList<SoftReference<FaultyOutputStream>>();
764 private long faultPos = 200;
765 AtomicBoolean fault = new AtomicBoolean(true);
766
767 public FaultyFileSystem() {
768 super(new LocalFileSystem());
769 System.err.println("Creating faulty!");
770 }
771
772 @Override
773 public FSDataOutputStream create(Path p) throws IOException {
774 return new FaultyOutputStream(super.create(p), faultPos, fault);
775 }
776
777 @Override
778 public FSDataOutputStream create(Path f, FsPermission permission,
779 boolean overwrite, int bufferSize, short replication, long blockSize,
780 Progressable progress) throws IOException {
781 return new FaultyOutputStream(super.create(f, permission,
782 overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
783 }
784
785 public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
786 int bufferSize, short replication, long blockSize, Progressable progress)
787 throws IOException {
788
789
790 return create(f, overwrite, bufferSize, replication, blockSize, progress);
791 }
792 }
793
794 static class FaultyOutputStream extends FSDataOutputStream {
795 volatile long faultPos = Long.MAX_VALUE;
796 private final AtomicBoolean fault;
797
798 public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
799 throws IOException {
800 super(out, null);
801 this.faultPos = faultPos;
802 this.fault = fault;
803 }
804
805 @Override
806 public void write(byte[] buf, int offset, int length) throws IOException {
807 System.err.println("faulty stream write at pos " + getPos());
808 injectFault();
809 super.write(buf, offset, length);
810 }
811
812 private void injectFault() throws IOException {
813 if (this.fault.get() && getPos() >= faultPos) {
814 throw new IOException("Fault injected");
815 }
816 }
817 }
818
819 private static void flushStore(HStore store, long id) throws IOException {
820 StoreFlushContext storeFlushCtx = store.createFlushContext(id);
821 storeFlushCtx.prepare();
822 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
823 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
824 }
825
826
827
828
829
830
831
832
833
834 List<Cell> getKeyValueSet(long[] timestamps, int numRows,
835 byte[] qualifier, byte[] family) {
836 List<Cell> kvList = new ArrayList<Cell>();
837 for (int i=1;i<=numRows;i++) {
838 byte[] b = Bytes.toBytes(i);
839 for (long timestamp: timestamps) {
840 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
841 }
842 }
843 return kvList;
844 }
845
846
847
848
849
850 @Test
851 public void testMultipleTimestamps() throws IOException {
852 int numRows = 1;
853 long[] timestamps1 = new long[] {1,5,10,20};
854 long[] timestamps2 = new long[] {30,80};
855
856 init(this.name.getMethodName());
857
858 List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
859 for (Cell kv : kvList1) {
860 this.store.add(KeyValueUtil.ensureKeyValue(kv));
861 }
862
863 this.store.snapshot();
864 flushStore(store, id++);
865
866 List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
867 for(Cell kv : kvList2) {
868 this.store.add(KeyValueUtil.ensureKeyValue(kv));
869 }
870
871 List<Cell> result;
872 Get get = new Get(Bytes.toBytes(1));
873 get.addColumn(family,qf1);
874
875 get.setTimeRange(0,15);
876 result = HBaseTestingUtility.getFromStoreFile(store, get);
877 Assert.assertTrue(result.size()>0);
878
879 get.setTimeRange(40,90);
880 result = HBaseTestingUtility.getFromStoreFile(store, get);
881 Assert.assertTrue(result.size()>0);
882
883 get.setTimeRange(10,45);
884 result = HBaseTestingUtility.getFromStoreFile(store, get);
885 Assert.assertTrue(result.size()>0);
886
887 get.setTimeRange(80,145);
888 result = HBaseTestingUtility.getFromStoreFile(store, get);
889 Assert.assertTrue(result.size()>0);
890
891 get.setTimeRange(1,2);
892 result = HBaseTestingUtility.getFromStoreFile(store, get);
893 Assert.assertTrue(result.size()>0);
894
895 get.setTimeRange(90,200);
896 result = HBaseTestingUtility.getFromStoreFile(store, get);
897 Assert.assertTrue(result.size()==0);
898 }
899
900
901
902
903
904
905 @Test
906 public void testSplitWithEmptyColFam() throws IOException {
907 init(this.name.getMethodName());
908 Assert.assertNull(store.getSplitPoint());
909 store.getHRegion().forceSplit(null);
910 Assert.assertNull(store.getSplitPoint());
911 store.getHRegion().clearSplit_TESTS_ONLY();
912 }
913
914 @Test
915 public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
916 final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
917 long anyValue = 10;
918
919
920
921
922 Configuration conf = HBaseConfiguration.create();
923 conf.setLong(CONFIG_KEY, anyValue);
924 init(name.getMethodName() + "-xml", conf);
925 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
926 Assert.assertFalse(store.throttleCompaction(anyValue));
927
928
929 --anyValue;
930 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
931 HColumnDescriptor hcd = new HColumnDescriptor(family);
932 htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
933 init(name.getMethodName() + "-htd", conf, htd, hcd);
934 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
935 Assert.assertFalse(store.throttleCompaction(anyValue));
936
937
938 --anyValue;
939 hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
940 init(name.getMethodName() + "-hcd", conf, htd, hcd);
941 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
942 Assert.assertFalse(store.throttleCompaction(anyValue));
943 }
944
945 public static class DummyStoreEngine extends DefaultStoreEngine {
946 public static DefaultCompactor lastCreatedCompactor = null;
947 @Override
948 protected void createComponents(
949 Configuration conf, Store store, KVComparator comparator) throws IOException {
950 super.createComponents(conf, store, comparator);
951 lastCreatedCompactor = this.compactor;
952 }
953 }
954
955 @Test
956 public void testStoreUsesSearchEngineOverride() throws Exception {
957 Configuration conf = HBaseConfiguration.create();
958 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
959 init(this.name.getMethodName(), conf);
960 Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
961 this.store.storeEngine.getCompactor());
962 }
963 }