1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import static org.junit.Assert.*;
23 import static org.mockito.Matchers.any;
24 import static org.mockito.Mockito.spy;
25 import static org.mockito.Mockito.times;
26 import static org.mockito.Mockito.verify;
27
28 import java.io.IOException;
29 import java.lang.ref.SoftReference;
30 import java.security.PrivilegedExceptionAction;
31 import java.util.ArrayList;
32 import java.util.Collection;
33 import java.util.Collections;
34 import java.util.Iterator;
35 import java.util.List;
36 import java.util.NavigableSet;
37 import java.util.concurrent.ConcurrentSkipListSet;
38 import java.util.concurrent.atomic.AtomicBoolean;
39
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.fs.FSDataOutputStream;
44 import org.apache.hadoop.fs.FileStatus;
45 import org.apache.hadoop.fs.FileSystem;
46 import org.apache.hadoop.fs.FilterFileSystem;
47 import org.apache.hadoop.fs.LocalFileSystem;
48 import org.apache.hadoop.fs.Path;
49 import org.apache.hadoop.fs.permission.FsPermission;
50 import org.apache.hadoop.hbase.Cell;
51 import org.apache.hadoop.hbase.CellUtil;
52 import org.apache.hadoop.hbase.HBaseConfiguration;
53 import org.apache.hadoop.hbase.HBaseTestingUtility;
54 import org.apache.hadoop.hbase.HColumnDescriptor;
55 import org.apache.hadoop.hbase.HRegionInfo;
56 import org.apache.hadoop.hbase.HTableDescriptor;
57 import org.apache.hadoop.hbase.KeyValue;
58 import org.apache.hadoop.hbase.KeyValue.KVComparator;
59 import org.apache.hadoop.hbase.KeyValueUtil;
60 import org.apache.hadoop.hbase.testclassification.MediumTests;
61 import org.apache.hadoop.hbase.TableName;
62 import org.apache.hadoop.hbase.client.Get;
63 import org.apache.hadoop.hbase.io.compress.Compression;
64 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
65 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
66 import org.apache.hadoop.hbase.io.hfile.HFile;
67 import org.apache.hadoop.hbase.io.hfile.HFileContext;
68 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
69 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
70 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
71 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
72 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
73 import org.apache.hadoop.hbase.wal.WALFactory;
74 import org.apache.hadoop.hbase.security.User;
75 import org.apache.hadoop.hbase.util.Bytes;
76 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
77 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
78 import org.apache.hadoop.hbase.util.FSUtils;
79 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
80 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
81 import org.apache.hadoop.util.Progressable;
82 import org.junit.After;
83 import org.junit.Assert;
84 import org.junit.Before;
85 import org.junit.Rule;
86 import org.junit.Test;
87 import org.junit.experimental.categories.Category;
88 import org.junit.rules.TestName;
89 import org.mockito.Mockito;
90
91 import com.google.common.collect.Lists;
92
93
94
95
96 @Category(MediumTests.class)
97 public class TestStore {
98 public static final Log LOG = LogFactory.getLog(TestStore.class);
99 @Rule public TestName name = new TestName();
100
101 HStore store;
102 byte [] table = Bytes.toBytes("table");
103 byte [] family = Bytes.toBytes("family");
104
105 byte [] row = Bytes.toBytes("row");
106 byte [] row2 = Bytes.toBytes("row2");
107 byte [] qf1 = Bytes.toBytes("qf1");
108 byte [] qf2 = Bytes.toBytes("qf2");
109 byte [] qf3 = Bytes.toBytes("qf3");
110 byte [] qf4 = Bytes.toBytes("qf4");
111 byte [] qf5 = Bytes.toBytes("qf5");
112 byte [] qf6 = Bytes.toBytes("qf6");
113
114 NavigableSet<byte[]> qualifiers =
115 new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
116
117 List<Cell> expected = new ArrayList<Cell>();
118 List<Cell> result = new ArrayList<Cell>();
119
120 long id = System.currentTimeMillis();
121 Get get = new Get(row);
122
123 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
124 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
125
126
127
128
129
130
131 @Before
132 public void setUp() throws IOException {
133 qualifiers.add(qf1);
134 qualifiers.add(qf3);
135 qualifiers.add(qf5);
136
137 Iterator<byte[]> iter = qualifiers.iterator();
138 while(iter.hasNext()){
139 byte [] next = iter.next();
140 expected.add(new KeyValue(row, family, next, 1, (byte[])null));
141 get.addColumn(family, next);
142 }
143 }
144
145 private void init(String methodName) throws IOException {
146 init(methodName, TEST_UTIL.getConfiguration());
147 }
148
149 private void init(String methodName, Configuration conf)
150 throws IOException {
151 HColumnDescriptor hcd = new HColumnDescriptor(family);
152
153
154 hcd.setMaxVersions(4);
155 init(methodName, conf, hcd);
156 }
157
158 private void init(String methodName, Configuration conf,
159 HColumnDescriptor hcd) throws IOException {
160 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
161 init(methodName, conf, htd, hcd);
162 }
163
164 @SuppressWarnings("deprecation")
165 private Store init(String methodName, Configuration conf, HTableDescriptor htd,
166 HColumnDescriptor hcd) throws IOException {
167
168 Path basedir = new Path(DIR+methodName);
169 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
170 final Path logdir = new Path(basedir, DefaultWALProvider.getWALDirectoryName(methodName));
171
172 FileSystem fs = FileSystem.get(conf);
173
174 fs.delete(logdir, true);
175
176 if (htd.hasFamily(hcd.getName())) {
177 htd.modifyFamily(hcd);
178 } else {
179 htd.addFamily(hcd);
180 }
181 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
182 final Configuration walConf = new Configuration(conf);
183 FSUtils.setRootDir(walConf, basedir);
184 final WALFactory wals = new WALFactory(walConf, null, methodName);
185 HRegion region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes()), fs, conf,
186 info, htd, null);
187
188 store = new HStore(region, hcd, conf);
189 return store;
190 }
191
192
193
194
195
196
197 @Test
198 public void testFlushSizeAccounting() throws Exception {
199 LOG.info("Setting up a faulty file system that cannot write in " +
200 this.name.getMethodName());
201 final Configuration conf = HBaseConfiguration.create();
202
203 conf.setInt("hbase.hstore.flush.retries.number", 1);
204 User user = User.createUserForTesting(conf, this.name.getMethodName(),
205 new String[]{"foo"});
206
207 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
208 user.runAs(new PrivilegedExceptionAction<Object>() {
209 @Override
210 public Object run() throws Exception {
211
212 FileSystem fs = FileSystem.get(conf);
213 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
214 FaultyFileSystem ffs = (FaultyFileSystem)fs;
215
216
217 init(name.getMethodName(), conf);
218
219 long size = store.memstore.getFlushableSize();
220 Assert.assertEquals(0, size);
221 LOG.info("Adding some data");
222 long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null)).getFirst();
223 size = store.memstore.getFlushableSize();
224 Assert.assertEquals(kvSize, size);
225
226 try {
227 LOG.info("Flushing");
228 flushStore(store, id++);
229 Assert.fail("Didn't bubble up IOE!");
230 } catch (IOException ioe) {
231 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
232 }
233 size = store.memstore.getFlushableSize();
234 Assert.assertEquals(kvSize, size);
235 store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
236
237
238 Assert.assertEquals(kvSize, size);
239 ffs.fault.set(false);
240 flushStore(store, id++);
241 size = store.memstore.getFlushableSize();
242
243 Assert.assertEquals(kvSize, size);
244 flushStore(store, id++);
245 size = store.memstore.getFlushableSize();
246 Assert.assertEquals(0, size);
247 return null;
248 }
249 });
250 }
251
252
253
254
255
256 @Test
257 public void testCreateWriter() throws Exception {
258 Configuration conf = HBaseConfiguration.create();
259 FileSystem fs = FileSystem.get(conf);
260
261 HColumnDescriptor hcd = new HColumnDescriptor(family);
262 hcd.setCompressionType(Compression.Algorithm.GZ);
263 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
264 init(name.getMethodName(), conf, hcd);
265
266
267 StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
268 Path path = writer.getPath();
269 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
270 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
271 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
272 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
273 writer.close();
274
275
276 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
277 Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
278 Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
279 reader.close();
280 }
281
282 @Test
283 public void testDeleteExpiredStoreFiles() throws Exception {
284 testDeleteExpiredStoreFiles(0);
285 testDeleteExpiredStoreFiles(1);
286 }
287
288
289
290
291 public void testDeleteExpiredStoreFiles(int minVersions) throws Exception {
292 int storeFileNum = 4;
293 int ttl = 4;
294 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
295 EnvironmentEdgeManagerTestHelper.injectEdge(edge);
296
297 Configuration conf = HBaseConfiguration.create();
298
299 conf.setBoolean("hbase.store.delete.expired.storefile", true);
300
301 conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 5);
302
303 HColumnDescriptor hcd = new HColumnDescriptor(family);
304 hcd.setMinVersions(minVersions);
305 hcd.setTimeToLive(ttl);
306 init(name.getMethodName(), conf, hcd);
307
308 long storeTtl = this.store.getScanInfo().getTtl();
309 long sleepTime = storeTtl / storeFileNum;
310 long timeStamp;
311
312
313 for (int i = 1; i <= storeFileNum; i++) {
314 LOG.info("Adding some data for the store file #" + i);
315 timeStamp = EnvironmentEdgeManager.currentTime();
316 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
317 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
318 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
319 flush(i);
320 edge.incrementTime(sleepTime);
321 }
322
323
324 Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
325
326
327
328 for (int i = 1; i <= storeFileNum - 1; i++) {
329
330 assertNull(this.store.requestCompaction());
331 Collection<StoreFile> sfs = this.store.getStorefiles();
332
333 if (minVersions == 0) {
334 assertEquals(storeFileNum - i, sfs.size());
335
336 for (StoreFile sf : sfs) {
337 assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTime() - storeTtl));
338 }
339 } else {
340 assertEquals(storeFileNum, sfs.size());
341 }
342
343 edge.incrementTime(sleepTime);
344 }
345 assertNull(this.store.requestCompaction());
346 Collection<StoreFile> sfs = this.store.getStorefiles();
347
348 if (minVersions == 0) {
349 assertEquals(1, sfs.size());
350 }
351 long ts = sfs.iterator().next().getReader().getMaxTimestamp();
352 assertTrue(ts < (edge.currentTime() - storeTtl));
353 }
354
355 @Test
356 public void testLowestModificationTime() throws Exception {
357 Configuration conf = HBaseConfiguration.create();
358 FileSystem fs = FileSystem.get(conf);
359
360 init(name.getMethodName(), conf);
361
362 int storeFileNum = 4;
363 for (int i = 1; i <= storeFileNum; i++) {
364 LOG.info("Adding some data for the store file #"+i);
365 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
366 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
367 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
368 flush(i);
369 }
370
371 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
372 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
373 Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
374
375
376 store.compact(store.requestCompaction());
377 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
378 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
379 Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
380 }
381
382 private static long getLowestTimeStampFromFS(FileSystem fs,
383 final Collection<StoreFile> candidates) throws IOException {
384 long minTs = Long.MAX_VALUE;
385 if (candidates.isEmpty()) {
386 return minTs;
387 }
388 Path[] p = new Path[candidates.size()];
389 int i = 0;
390 for (StoreFile sf : candidates) {
391 p[i] = sf.getPath();
392 ++i;
393 }
394
395 FileStatus[] stats = fs.listStatus(p);
396 if (stats == null || stats.length == 0) {
397 return minTs;
398 }
399 for (FileStatus s : stats) {
400 minTs = Math.min(minTs, s.getModificationTime());
401 }
402 return minTs;
403 }
404
405
406
407
408
409 private static final int BLOCKSIZE_SMALL = 8192;
410
411
412
413
414 @Test
415 public void testEmptyStoreFile() throws IOException {
416 init(this.name.getMethodName());
417
418 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
419 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
420 flush(1);
421
422
423 StoreFile f = this.store.getStorefiles().iterator().next();
424 Path storedir = f.getPath().getParent();
425 long seqid = f.getMaxSequenceId();
426 Configuration c = HBaseConfiguration.create();
427 FileSystem fs = FileSystem.get(c);
428 HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
429 StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
430 fs)
431 .withOutputDir(storedir)
432 .withFileContext(meta)
433 .build();
434 w.appendMetadata(seqid + 1, false);
435 w.close();
436 this.store.close();
437
438 this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
439 Assert.assertEquals(2, this.store.getStorefilesCount());
440
441 result = HBaseTestingUtility.getFromStoreFile(store,
442 get.getRow(),
443 qualifiers);
444 Assert.assertEquals(1, result.size());
445 }
446
447
448
449
450
451 @Test
452 public void testGet_FromMemStoreOnly() throws IOException {
453 init(this.name.getMethodName());
454
455
456 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
457 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
458 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
459 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
460 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
461 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
462
463
464 result = HBaseTestingUtility.getFromStoreFile(store,
465 get.getRow(), qualifiers);
466
467
468 assertCheck();
469 }
470
471
472
473
474
475 @Test
476 public void testGet_FromFilesOnly() throws IOException {
477 init(this.name.getMethodName());
478
479
480 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
481 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
482
483 flush(1);
484
485
486 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
487 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
488
489 flush(2);
490
491
492 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
493 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
494
495 flush(3);
496
497
498 result = HBaseTestingUtility.getFromStoreFile(store,
499 get.getRow(),
500 qualifiers);
501
502
503
504 Collections.sort(result, KeyValue.COMPARATOR);
505
506
507 assertCheck();
508 }
509
510
511
512
513
514 @Test
515 public void testGet_FromMemStoreAndFiles() throws IOException {
516 init(this.name.getMethodName());
517
518
519 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
520 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
521
522 flush(1);
523
524
525 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
526 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
527
528 flush(2);
529
530
531 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
532 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
533
534
535 result = HBaseTestingUtility.getFromStoreFile(store,
536 get.getRow(), qualifiers);
537
538
539 Collections.sort(result, KeyValue.COMPARATOR);
540
541
542 assertCheck();
543 }
544
545 private void flush(int storeFilessize) throws IOException{
546 this.store.snapshot();
547 flushStore(store, id++);
548 Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
549 Assert.assertEquals(0, ((DefaultMemStore)this.store.memstore).cellSet.size());
550 }
551
552 private void assertCheck() {
553 Assert.assertEquals(expected.size(), result.size());
554 for(int i=0; i<expected.size(); i++) {
555 Assert.assertEquals(expected.get(i), result.get(i));
556 }
557 }
558
559
560
561
562
563
564
565 @Test
566 public void testIncrementColumnValue_ICVDuringFlush()
567 throws IOException, InterruptedException {
568 init(this.name.getMethodName());
569
570 long oldValue = 1L;
571 long newValue = 3L;
572 this.store.add(new KeyValue(row, family, qf1,
573 System.currentTimeMillis(),
574 Bytes.toBytes(oldValue)));
575
576
577 this.store.snapshot();
578
579
580 this.store.add(new KeyValue(row, family, qf2,
581 System.currentTimeMillis(),
582 Bytes.toBytes(oldValue)));
583
584
585 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
586
587
588 Assert.assertTrue(ret > 0);
589
590
591 flushStore(store, id++);
592 Assert.assertEquals(1, this.store.getStorefiles().size());
593
594 Assert.assertEquals(2, ((DefaultMemStore)this.store.memstore).cellSet.size());
595
596
597 Get get = new Get(row);
598 get.addColumn(family, qf1);
599 get.setMaxVersions();
600 List<Cell> results = new ArrayList<Cell>();
601
602 results = HBaseTestingUtility.getFromStoreFile(store, get);
603 Assert.assertEquals(2, results.size());
604
605 long ts1 = results.get(0).getTimestamp();
606 long ts2 = results.get(1).getTimestamp();
607
608 Assert.assertTrue(ts1 > ts2);
609
610 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
611 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
612 }
613
614 @After
615 public void tearDown() throws Exception {
616 EnvironmentEdgeManagerTestHelper.reset();
617 }
618
619 @Test
620 public void testICV_negMemstoreSize() throws IOException {
621 init(this.name.getMethodName());
622
623 long time = 100;
624 ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
625 ee.setValue(time);
626 EnvironmentEdgeManagerTestHelper.injectEdge(ee);
627 long newValue = 3L;
628 long size = 0;
629
630
631 size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
632 System.currentTimeMillis(),
633 Bytes.toBytes(newValue))).getFirst();
634 size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
635 System.currentTimeMillis(),
636 Bytes.toBytes(newValue))).getFirst();
637 size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
638 System.currentTimeMillis(),
639 Bytes.toBytes(newValue))).getFirst();
640 size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
641 System.currentTimeMillis(),
642 Bytes.toBytes(newValue))).getFirst();
643 size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
644 System.currentTimeMillis(),
645 Bytes.toBytes(newValue))).getFirst();
646
647
648 for ( int i = 0 ; i < 10000 ; ++i) {
649 newValue++;
650
651 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
652 long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
653
654 if (ret != 0) System.out.println("ret: " + ret);
655 if (ret2 != 0) System.out.println("ret2: " + ret2);
656
657 Assert.assertTrue("ret: " + ret, ret >= 0);
658 size += ret;
659 Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
660 size += ret2;
661
662
663 if (i % 1000 == 0)
664 ee.setValue(++time);
665 }
666
667 long computedSize=0;
668 for (Cell cell : ((DefaultMemStore)this.store.memstore).cellSet) {
669 long kvsize = DefaultMemStore.heapSizeChange(cell, true);
670
671 computedSize += kvsize;
672 }
673 Assert.assertEquals(computedSize, size);
674 }
675
676 @Test
677 public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
678 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
679 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
680 init(this.name.getMethodName());
681
682 long oldValue = 1L;
683 long newValue = 3L;
684 this.store.add(new KeyValue(row, family, qf1,
685 EnvironmentEdgeManager.currentTime(),
686 Bytes.toBytes(oldValue)));
687
688
689 this.store.snapshot();
690
691
692 long ret = this.store.updateColumnValue(row, family, qf1, newValue);
693
694
695 Assert.assertTrue(ret > 0);
696
697
698 flushStore(store, id++);
699 Assert.assertEquals(1, this.store.getStorefiles().size());
700 Assert.assertEquals(1, ((DefaultMemStore)this.store.memstore).cellSet.size());
701
702
703 newValue += 1;
704 this.store.updateColumnValue(row, family, qf1, newValue);
705
706
707 newValue += 1;
708 this.store.updateColumnValue(row, family, qf1, newValue);
709
710
711
712
713
714 Get get = new Get(row);
715 get.addColumn(family, qf1);
716 get.setMaxVersions();
717 List<Cell> results = new ArrayList<Cell>();
718
719 results = HBaseTestingUtility.getFromStoreFile(store, get);
720 Assert.assertEquals(2, results.size());
721
722 long ts1 = results.get(0).getTimestamp();
723 long ts2 = results.get(1).getTimestamp();
724
725 Assert.assertTrue(ts1 > ts2);
726 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
727 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
728
729 mee.setValue(2);
730 newValue += 1;
731 this.store.updateColumnValue(row, family, qf1, newValue);
732
733 results = HBaseTestingUtility.getFromStoreFile(store, get);
734 Assert.assertEquals(2, results.size());
735
736 ts1 = results.get(0).getTimestamp();
737 ts2 = results.get(1).getTimestamp();
738
739 Assert.assertTrue(ts1 > ts2);
740 Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
741 Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
742 }
743
744 @Test
745 public void testHandleErrorsInFlush() throws Exception {
746 LOG.info("Setting up a faulty file system that cannot write");
747
748 final Configuration conf = HBaseConfiguration.create();
749 User user = User.createUserForTesting(conf,
750 "testhandleerrorsinflush", new String[]{"foo"});
751
752 conf.setClass("fs.file.impl", FaultyFileSystem.class,
753 FileSystem.class);
754 user.runAs(new PrivilegedExceptionAction<Object>() {
755 @Override
756 public Object run() throws Exception {
757
758 FileSystem fs = FileSystem.get(conf);
759 Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
760
761
762 init(name.getMethodName(), conf);
763
764 LOG.info("Adding some data");
765 store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
766 store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
767 store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
768
769 LOG.info("Before flush, we should have no files");
770
771 Collection<StoreFileInfo> files =
772 store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
773 Assert.assertEquals(0, files != null ? files.size() : 0);
774
775
776 try {
777 LOG.info("Flushing");
778 flush(1);
779 Assert.fail("Didn't bubble up IOE!");
780 } catch (IOException ioe) {
781 Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
782 }
783
784 LOG.info("After failed flush, we should still have no files!");
785 files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
786 Assert.assertEquals(0, files != null ? files.size() : 0);
787 store.getHRegion().getWAL().close();
788 return null;
789 }
790 });
791 FileSystem.closeAllForUGI(user.getUGI());
792 }
793
794
795
796
797
798 static class FaultyFileSystem extends FilterFileSystem {
799 List<SoftReference<FaultyOutputStream>> outStreams =
800 new ArrayList<SoftReference<FaultyOutputStream>>();
801 private long faultPos = 200;
802 AtomicBoolean fault = new AtomicBoolean(true);
803
804 public FaultyFileSystem() {
805 super(new LocalFileSystem());
806 System.err.println("Creating faulty!");
807 }
808
809 @Override
810 public FSDataOutputStream create(Path p) throws IOException {
811 return new FaultyOutputStream(super.create(p), faultPos, fault);
812 }
813
814 @Override
815 public FSDataOutputStream create(Path f, FsPermission permission,
816 boolean overwrite, int bufferSize, short replication, long blockSize,
817 Progressable progress) throws IOException {
818 return new FaultyOutputStream(super.create(f, permission,
819 overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
820 }
821
822 @Override
823 public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
824 int bufferSize, short replication, long blockSize, Progressable progress)
825 throws IOException {
826
827
828 return create(f, overwrite, bufferSize, replication, blockSize, progress);
829 }
830 }
831
832 static class FaultyOutputStream extends FSDataOutputStream {
833 volatile long faultPos = Long.MAX_VALUE;
834 private final AtomicBoolean fault;
835
836 public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
837 throws IOException {
838 super(out, null);
839 this.faultPos = faultPos;
840 this.fault = fault;
841 }
842
843 @Override
844 public void write(byte[] buf, int offset, int length) throws IOException {
845 System.err.println("faulty stream write at pos " + getPos());
846 injectFault();
847 super.write(buf, offset, length);
848 }
849
850 private void injectFault() throws IOException {
851 if (this.fault.get() && getPos() >= faultPos) {
852 throw new IOException("Fault injected");
853 }
854 }
855 }
856
857 private static void flushStore(HStore store, long id) throws IOException {
858 StoreFlushContext storeFlushCtx = store.createFlushContext(id);
859 storeFlushCtx.prepare();
860 storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
861 storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
862 }
863
864
865
866
867
868
869
870
871
872 List<Cell> getKeyValueSet(long[] timestamps, int numRows,
873 byte[] qualifier, byte[] family) {
874 List<Cell> kvList = new ArrayList<Cell>();
875 for (int i=1;i<=numRows;i++) {
876 byte[] b = Bytes.toBytes(i);
877 for (long timestamp: timestamps) {
878 kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
879 }
880 }
881 return kvList;
882 }
883
884
885
886
887
888 @Test
889 public void testMultipleTimestamps() throws IOException {
890 int numRows = 1;
891 long[] timestamps1 = new long[] {1,5,10,20};
892 long[] timestamps2 = new long[] {30,80};
893
894 init(this.name.getMethodName());
895
896 List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
897 for (Cell kv : kvList1) {
898 this.store.add(KeyValueUtil.ensureKeyValue(kv));
899 }
900
901 this.store.snapshot();
902 flushStore(store, id++);
903
904 List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
905 for(Cell kv : kvList2) {
906 this.store.add(KeyValueUtil.ensureKeyValue(kv));
907 }
908
909 List<Cell> result;
910 Get get = new Get(Bytes.toBytes(1));
911 get.addColumn(family,qf1);
912
913 get.setTimeRange(0,15);
914 result = HBaseTestingUtility.getFromStoreFile(store, get);
915 Assert.assertTrue(result.size()>0);
916
917 get.setTimeRange(40,90);
918 result = HBaseTestingUtility.getFromStoreFile(store, get);
919 Assert.assertTrue(result.size()>0);
920
921 get.setTimeRange(10,45);
922 result = HBaseTestingUtility.getFromStoreFile(store, get);
923 Assert.assertTrue(result.size()>0);
924
925 get.setTimeRange(80,145);
926 result = HBaseTestingUtility.getFromStoreFile(store, get);
927 Assert.assertTrue(result.size()>0);
928
929 get.setTimeRange(1,2);
930 result = HBaseTestingUtility.getFromStoreFile(store, get);
931 Assert.assertTrue(result.size()>0);
932
933 get.setTimeRange(90,200);
934 result = HBaseTestingUtility.getFromStoreFile(store, get);
935 Assert.assertTrue(result.size()==0);
936 }
937
938
939
940
941
942
943 @Test
944 public void testSplitWithEmptyColFam() throws IOException {
945 init(this.name.getMethodName());
946 Assert.assertNull(store.getSplitPoint());
947 store.getHRegion().forceSplit(null);
948 Assert.assertNull(store.getSplitPoint());
949 store.getHRegion().clearSplit();
950 }
951
952 @Test
953 public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
954 final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
955 long anyValue = 10;
956
957
958
959
960 Configuration conf = HBaseConfiguration.create();
961 conf.setLong(CONFIG_KEY, anyValue);
962 init(name.getMethodName() + "-xml", conf);
963 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
964 Assert.assertFalse(store.throttleCompaction(anyValue));
965
966
967 --anyValue;
968 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
969 HColumnDescriptor hcd = new HColumnDescriptor(family);
970 htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
971 init(name.getMethodName() + "-htd", conf, htd, hcd);
972 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
973 Assert.assertFalse(store.throttleCompaction(anyValue));
974
975
976 --anyValue;
977 hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
978 init(name.getMethodName() + "-hcd", conf, htd, hcd);
979 Assert.assertTrue(store.throttleCompaction(anyValue + 1));
980 Assert.assertFalse(store.throttleCompaction(anyValue));
981 }
982
983 public static class DummyStoreEngine extends DefaultStoreEngine {
984 public static DefaultCompactor lastCreatedCompactor = null;
985 @Override
986 protected void createComponents(
987 Configuration conf, Store store, KVComparator comparator) throws IOException {
988 super.createComponents(conf, store, comparator);
989 lastCreatedCompactor = this.compactor;
990 }
991 }
992
993 @Test
994 public void testStoreUsesSearchEngineOverride() throws Exception {
995 Configuration conf = HBaseConfiguration.create();
996 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
997 init(this.name.getMethodName(), conf);
998 Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
999 this.store.storeEngine.getCompactor());
1000 }
1001
1002 private void addStoreFile() throws IOException {
1003 StoreFile f = this.store.getStorefiles().iterator().next();
1004 Path storedir = f.getPath().getParent();
1005 long seqid = this.store.getMaxSequenceId();
1006 Configuration c = TEST_UTIL.getConfiguration();
1007 FileSystem fs = FileSystem.get(c);
1008 HFileContext fileContext = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
1009 StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
1010 fs)
1011 .withOutputDir(storedir)
1012 .withFileContext(fileContext)
1013 .build();
1014 w.appendMetadata(seqid + 1, false);
1015 w.close();
1016 LOG.info("Added store file:" + w.getPath());
1017 }
1018
1019 private void archiveStoreFile(int index) throws IOException {
1020 Collection<StoreFile> files = this.store.getStorefiles();
1021 StoreFile sf = null;
1022 Iterator<StoreFile> it = files.iterator();
1023 for (int i = 0; i <= index; i++) {
1024 sf = it.next();
1025 }
1026 store.getRegionFileSystem().removeStoreFiles(store.getColumnFamilyName(), Lists.newArrayList(sf));
1027 }
1028
1029 @Test
1030 public void testRefreshStoreFiles() throws Exception {
1031 init(name.getMethodName());
1032
1033 assertEquals(0, this.store.getStorefilesCount());
1034
1035
1036 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
1037 flush(1);
1038 assertEquals(1, this.store.getStorefilesCount());
1039
1040
1041 addStoreFile();
1042
1043 assertEquals(1, this.store.getStorefilesCount());
1044 store.refreshStoreFiles();
1045 assertEquals(2, this.store.getStorefilesCount());
1046
1047
1048 addStoreFile();
1049 addStoreFile();
1050 addStoreFile();
1051
1052 assertEquals(2, this.store.getStorefilesCount());
1053 store.refreshStoreFiles();
1054 assertEquals(5, this.store.getStorefilesCount());
1055
1056 archiveStoreFile(0);
1057
1058 assertEquals(5, this.store.getStorefilesCount());
1059 store.refreshStoreFiles();
1060 assertEquals(4, this.store.getStorefilesCount());
1061
1062 archiveStoreFile(0);
1063 archiveStoreFile(1);
1064 archiveStoreFile(2);
1065
1066 assertEquals(4, this.store.getStorefilesCount());
1067 store.refreshStoreFiles();
1068 assertEquals(1, this.store.getStorefilesCount());
1069
1070 archiveStoreFile(0);
1071 store.refreshStoreFiles();
1072 assertEquals(0, this.store.getStorefilesCount());
1073 }
1074
1075 @SuppressWarnings("unchecked")
1076 @Test
1077 public void testRefreshStoreFilesNotChanged() throws IOException {
1078 init(name.getMethodName());
1079
1080 assertEquals(0, this.store.getStorefilesCount());
1081
1082
1083 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
1084 flush(1);
1085
1086 addStoreFile();
1087
1088 HStore spiedStore = spy(store);
1089
1090
1091 spiedStore.refreshStoreFiles();
1092 assertEquals(2, this.store.getStorefilesCount());
1093 verify(spiedStore, times(1)).replaceStoreFiles(any(Collection.class), any(Collection.class));
1094
1095
1096 spiedStore.refreshStoreFiles();
1097
1098
1099 verify(spiedStore, times(0)).replaceStoreFiles(null, null);
1100 }
1101 }