View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.lang.ref.SoftReference;
24  import java.security.PrivilegedExceptionAction;
25  import java.util.ArrayList;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.NavigableSet;
31  import java.util.concurrent.ConcurrentSkipListSet;
32  import java.util.concurrent.atomic.AtomicBoolean;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FSDataOutputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.FilterFileSystem;
41  import org.apache.hadoop.fs.LocalFileSystem;
42  import org.apache.hadoop.fs.Path;
43  import org.apache.hadoop.fs.permission.FsPermission;
44  import org.apache.hadoop.hbase.Cell;
45  import org.apache.hadoop.hbase.CellUtil;
46  import org.apache.hadoop.hbase.HBaseConfiguration;
47  import org.apache.hadoop.hbase.HBaseTestingUtility;
48  import org.apache.hadoop.hbase.HColumnDescriptor;
49  import org.apache.hadoop.hbase.HRegionInfo;
50  import org.apache.hadoop.hbase.HTableDescriptor;
51  import org.apache.hadoop.hbase.KeyValue;
52  import org.apache.hadoop.hbase.KeyValue.KVComparator;
53  import org.apache.hadoop.hbase.KeyValueUtil;
54  import org.apache.hadoop.hbase.testclassification.MediumTests;
55  import org.apache.hadoop.hbase.TableName;
56  import org.apache.hadoop.hbase.client.Get;
57  import org.apache.hadoop.hbase.io.compress.Compression;
58  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
59  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
60  import org.apache.hadoop.hbase.io.hfile.HFile;
61  import org.apache.hadoop.hbase.io.hfile.HFileContext;
62  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
63  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
64  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
65  import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
66  import org.apache.hadoop.hbase.regionserver.wal.HLog;
67  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
68  import org.apache.hadoop.hbase.security.User;
69  import org.apache.hadoop.hbase.util.Bytes;
70  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
71  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
72  import org.apache.hadoop.hbase.util.FSUtils;
73  import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
74  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
75  import org.apache.hadoop.util.Progressable;
76  import org.junit.After;
77  import org.junit.Assert;
78  import org.junit.Before;
79  import org.junit.Rule;
80  import org.junit.Test;
81  import org.junit.experimental.categories.Category;
82  import org.junit.rules.TestName;
83  import org.mockito.Mockito;
84  
85  /**
86   * Test class for the Store
87   */
88  @Category(MediumTests.class)
89  public class TestStore {
90    public static final Log LOG = LogFactory.getLog(TestStore.class);
91    @Rule public TestName name = new TestName();
92  
93    HStore store;
94    byte [] table = Bytes.toBytes("table");
95    byte [] family = Bytes.toBytes("family");
96  
97    byte [] row = Bytes.toBytes("row");
98    byte [] row2 = Bytes.toBytes("row2");
99    byte [] qf1 = Bytes.toBytes("qf1");
100   byte [] qf2 = Bytes.toBytes("qf2");
101   byte [] qf3 = Bytes.toBytes("qf3");
102   byte [] qf4 = Bytes.toBytes("qf4");
103   byte [] qf5 = Bytes.toBytes("qf5");
104   byte [] qf6 = Bytes.toBytes("qf6");
105 
106   NavigableSet<byte[]> qualifiers =
107     new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
108 
109   List<Cell> expected = new ArrayList<Cell>();
110   List<Cell> result = new ArrayList<Cell>();
111 
112   long id = System.currentTimeMillis();
113   Get get = new Get(row);
114 
115   private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
116   private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
117 
118 
119   /**
120    * Setup
121    * @throws IOException
122    */
123   @Before
124   public void setUp() throws IOException {
125     qualifiers.add(qf1);
126     qualifiers.add(qf3);
127     qualifiers.add(qf5);
128 
129     Iterator<byte[]> iter = qualifiers.iterator();
130     while(iter.hasNext()){
131       byte [] next = iter.next();
132       expected.add(new KeyValue(row, family, next, 1, (byte[])null));
133       get.addColumn(family, next);
134     }
135   }
136 
137   private void init(String methodName) throws IOException {
138     init(methodName, HBaseConfiguration.create());
139   }
140 
141   private void init(String methodName, Configuration conf)
142   throws IOException {
143     HColumnDescriptor hcd = new HColumnDescriptor(family);
144     // some of the tests write 4 versions and then flush
145     // (with HBASE-4241, lower versions are collected on flush)
146     hcd.setMaxVersions(4);
147     init(methodName, conf, hcd);
148   }
149 
150   private void init(String methodName, Configuration conf,
151       HColumnDescriptor hcd) throws IOException {
152     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
153     init(methodName, conf, htd, hcd);
154   }
155 
156   @SuppressWarnings("deprecation")
157   private Store init(String methodName, Configuration conf, HTableDescriptor htd,
158       HColumnDescriptor hcd) throws IOException {
159     //Setting up a Store
160     Path basedir = new Path(DIR+methodName);
161     Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
162     String logName = "logs";
163     Path logdir = new Path(basedir, logName);
164 
165     FileSystem fs = FileSystem.get(conf);
166 
167     fs.delete(logdir, true);
168 
169     htd.addFamily(hcd);
170     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
171     HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
172     HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
173 
174     store = new HStore(region, hcd, conf);
175     return store;
176   }
177 
178   /**
179    * Test we do not lose data if we fail a flush and then close.
180    * Part of HBase-10466
181    * @throws Exception
182    */
183   @Test
184   public void testFlushSizeAccounting() throws Exception {
185     LOG.info("Setting up a faulty file system that cannot write in " +
186       this.name.getMethodName());
187     final Configuration conf = HBaseConfiguration.create();
188     // Only retry once.
189     conf.setInt("hbase.hstore.flush.retries.number", 1);
190     User user = User.createUserForTesting(conf, this.name.getMethodName(),
191       new String[]{"foo"});
192     // Inject our faulty LocalFileSystem
193     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
194     user.runAs(new PrivilegedExceptionAction<Object>() {
195       public Object run() throws Exception {
196         // Make sure it worked (above is sensitive to caching details in hadoop core)
197         FileSystem fs = FileSystem.get(conf);
198         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
199         FaultyFileSystem ffs = (FaultyFileSystem)fs;
200 
201         // Initialize region
202         init(name.getMethodName(), conf);
203 
204         long size = store.memstore.getFlushableSize();
205         Assert.assertEquals(0, size);
206         LOG.info("Adding some data");
207         long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
208         size = store.memstore.getFlushableSize();
209         Assert.assertEquals(kvSize, size);
210         // Flush.  Bug #1 from HBASE-10466.  Make sure size calculation on failed flush is right.
211         try {
212           LOG.info("Flushing");
213           flushStore(store, id++);
214           Assert.fail("Didn't bubble up IOE!");
215         } catch (IOException ioe) {
216           Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
217         }
218         size = store.memstore.getFlushableSize();
219         Assert.assertEquals(kvSize, size);
220         store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
221         // Even though we add a new kv, we expect the flushable size to be 'same' since we have
222         // not yet cleared the snapshot -- the above flush failed.
223         Assert.assertEquals(kvSize, size);
224         ffs.fault.set(false);
225         flushStore(store, id++);
226         size = store.memstore.getFlushableSize();
227         // Size should be the foreground kv size.
228         Assert.assertEquals(kvSize, size);
229         flushStore(store, id++);
230         size = store.memstore.getFlushableSize();
231         Assert.assertEquals(0, size);
232         return null;
233       }
234     });
235   }
236 
237   /**
238    * Verify that compression and data block encoding are respected by the
239    * Store.createWriterInTmp() method, used on store flush.
240    */
241   @Test
242   public void testCreateWriter() throws Exception {
243     Configuration conf = HBaseConfiguration.create();
244     FileSystem fs = FileSystem.get(conf);
245 
246     HColumnDescriptor hcd = new HColumnDescriptor(family);
247     hcd.setCompressionType(Compression.Algorithm.GZ);
248     hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
249     init(name.getMethodName(), conf, hcd);
250 
251     // Test createWriterInTmp()
252     StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
253     Path path = writer.getPath();
254     writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
255     writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
256     writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
257     writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
258     writer.close();
259 
260     // Verify that compression and encoding settings are respected
261     HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
262     Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
263     Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
264     reader.close();
265   }
266 
267   @Test
268   public void testDeleteExpiredStoreFiles() throws Exception {
269     testDeleteExpiredStoreFiles(0);
270     testDeleteExpiredStoreFiles(1);
271   }
272 
273   /*
274    * @param minVersions the MIN_VERSIONS for the column family
275    */
276   public void testDeleteExpiredStoreFiles(int minVersions) throws Exception {
277     int storeFileNum = 4;
278     int ttl = 4;
279     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
280     EnvironmentEdgeManagerTestHelper.injectEdge(edge);
281     
282     Configuration conf = HBaseConfiguration.create();
283     // Enable the expired store file deletion
284     conf.setBoolean("hbase.store.delete.expired.storefile", true);
285     // Set the compaction threshold higher to avoid normal compactions.
286     conf.setInt(CompactionConfiguration.MIN_KEY, 5);
287 
288     HColumnDescriptor hcd = new HColumnDescriptor(family);
289     hcd.setMinVersions(minVersions);
290     hcd.setTimeToLive(ttl);
291     init(name.getMethodName(), conf, hcd);
292 
293     long storeTtl = this.store.getScanInfo().getTtl();
294     long sleepTime = storeTtl / storeFileNum;
295     long timeStamp;
296     // There are 4 store files and the max time stamp difference among these
297     // store files will be (this.store.ttl / storeFileNum)
298     for (int i = 1; i <= storeFileNum; i++) {
299       LOG.info("Adding some data for the store file #" + i);
300       timeStamp = EnvironmentEdgeManager.currentTimeMillis();
301       this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
302       this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
303       this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
304       flush(i);
305       edge.incrementTime(sleepTime);
306     }
307 
308     // Verify the total number of store files
309     Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
310 
311      // Each call will find one expired store file and delete it before compaction happens.
312      // There will be no compaction due to threshold above. Last file will not be replaced.
313     for (int i = 1; i <= storeFileNum - 1; i++) {
314       // verify the expired store file.
315       Assert.assertNull(this.store.requestCompaction());
316       Collection<StoreFile> sfs = this.store.getStorefiles();
317       // Ensure i files are gone.
318       if (minVersions == 0) {
319         Assert.assertEquals(storeFileNum - i, sfs.size());
320         // Ensure only non-expired files remain.
321         for (StoreFile sf : sfs) {
322           Assert.assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTimeMillis() - storeTtl));
323         }
324       } else {
325         Assert.assertEquals(storeFileNum, sfs.size());
326       }
327       // Let the next store file expired.
328       edge.incrementTime(sleepTime);
329     }
330     Assert.assertNull(this.store.requestCompaction());
331     Collection<StoreFile> sfs = this.store.getStorefiles();
332     // Assert the last expired file is not removed.
333     if (minVersions == 0) {
334       Assert.assertEquals(1, sfs.size());
335     }
336     long ts = sfs.iterator().next().getReader().getMaxTimestamp();
337     Assert.assertTrue(ts < (edge.currentTimeMillis() - storeTtl));
338   }
339 
340   @Test
341   public void testLowestModificationTime() throws Exception {
342     Configuration conf = HBaseConfiguration.create();
343     FileSystem fs = FileSystem.get(conf);
344     // Initialize region
345     init(name.getMethodName(), conf);
346     
347     int storeFileNum = 4;
348     for (int i = 1; i <= storeFileNum; i++) {
349       LOG.info("Adding some data for the store file #"+i);
350       this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
351       this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
352       this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
353       flush(i);
354     }
355     // after flush; check the lowest time stamp
356     long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
357     long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
358     Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
359 
360     // after compact; check the lowest time stamp
361     store.compact(store.requestCompaction());
362     lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
363     lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
364     Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
365   }
366   
367   private static long getLowestTimeStampFromFS(FileSystem fs, 
368       final Collection<StoreFile> candidates) throws IOException {
369     long minTs = Long.MAX_VALUE;
370     if (candidates.isEmpty()) {
371       return minTs; 
372     }
373     Path[] p = new Path[candidates.size()];
374     int i = 0;
375     for (StoreFile sf : candidates) {
376       p[i] = sf.getPath();
377       ++i;
378     }
379     
380     FileStatus[] stats = fs.listStatus(p);
381     if (stats == null || stats.length == 0) {
382       return minTs;
383     }
384     for (FileStatus s : stats) {
385       minTs = Math.min(minTs, s.getModificationTime());
386     }
387     return minTs;
388   }
389 
390   //////////////////////////////////////////////////////////////////////////////
391   // Get tests
392   //////////////////////////////////////////////////////////////////////////////
393 
394   private static final int BLOCKSIZE_SMALL = 8192;
395   /**
396    * Test for hbase-1686.
397    * @throws IOException
398    */
399   @Test
400   public void testEmptyStoreFile() throws IOException {
401     init(this.name.getMethodName());
402     // Write a store file.
403     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
404     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
405     flush(1);
406     // Now put in place an empty store file.  Its a little tricky.  Have to
407     // do manually with hacked in sequence id.
408     StoreFile f = this.store.getStorefiles().iterator().next();
409     Path storedir = f.getPath().getParent();
410     long seqid = f.getMaxSequenceId();
411     Configuration c = HBaseConfiguration.create();
412     FileSystem fs = FileSystem.get(c);
413     HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
414     StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
415         fs)
416             .withOutputDir(storedir)
417             .withFileContext(meta)
418             .build();
419     w.appendMetadata(seqid + 1, false);
420     w.close();
421     this.store.close();
422     // Reopen it... should pick up two files
423     this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
424     Assert.assertEquals(2, this.store.getStorefilesCount());
425 
426     result = HBaseTestingUtility.getFromStoreFile(store,
427         get.getRow(),
428         qualifiers);
429     Assert.assertEquals(1, result.size());
430   }
431 
432   /**
433    * Getting data from memstore only
434    * @throws IOException
435    */
436   @Test
437   public void testGet_FromMemStoreOnly() throws IOException {
438     init(this.name.getMethodName());
439 
440     //Put data in memstore
441     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
442     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
443     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
444     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
445     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
446     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
447 
448     //Get
449     result = HBaseTestingUtility.getFromStoreFile(store,
450         get.getRow(), qualifiers);
451 
452     //Compare
453     assertCheck();
454   }
455 
456   /**
457    * Getting data from files only
458    * @throws IOException
459    */
460   @Test
461   public void testGet_FromFilesOnly() throws IOException {
462     init(this.name.getMethodName());
463 
464     //Put data in memstore
465     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
466     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
467     //flush
468     flush(1);
469 
470     //Add more data
471     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
472     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
473     //flush
474     flush(2);
475 
476     //Add more data
477     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
478     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
479     //flush
480     flush(3);
481 
482     //Get
483     result = HBaseTestingUtility.getFromStoreFile(store,
484         get.getRow(),
485         qualifiers);
486     //this.store.get(get, qualifiers, result);
487 
488     //Need to sort the result since multiple files
489     Collections.sort(result, KeyValue.COMPARATOR);
490 
491     //Compare
492     assertCheck();
493   }
494 
495   /**
496    * Getting data from memstore and files
497    * @throws IOException
498    */
499   @Test
500   public void testGet_FromMemStoreAndFiles() throws IOException {
501     init(this.name.getMethodName());
502 
503     //Put data in memstore
504     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
505     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
506     //flush
507     flush(1);
508 
509     //Add more data
510     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
511     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
512     //flush
513     flush(2);
514 
515     //Add more data
516     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
517     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
518 
519     //Get
520     result = HBaseTestingUtility.getFromStoreFile(store,
521         get.getRow(), qualifiers);
522 
523     //Need to sort the result since multiple files
524     Collections.sort(result, KeyValue.COMPARATOR);
525 
526     //Compare
527     assertCheck();
528   }
529 
530   private void flush(int storeFilessize) throws IOException{
531     this.store.snapshot();
532     flushStore(store, id++);
533     Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
534     Assert.assertEquals(0, this.store.memstore.kvset.size());
535   }
536 
537   private void assertCheck() {
538     Assert.assertEquals(expected.size(), result.size());
539     for(int i=0; i<expected.size(); i++) {
540       Assert.assertEquals(expected.get(i), result.get(i));
541     }
542   }
543 
544   //////////////////////////////////////////////////////////////////////////////
545   // IncrementColumnValue tests
546   //////////////////////////////////////////////////////////////////////////////
547   /*
548    * test the internal details of how ICV works, especially during a flush scenario.
549    */
550   @Test
551   public void testIncrementColumnValue_ICVDuringFlush()
552       throws IOException, InterruptedException {
553     init(this.name.getMethodName());
554 
555     long oldValue = 1L;
556     long newValue = 3L;
557     this.store.add(new KeyValue(row, family, qf1,
558         System.currentTimeMillis(),
559         Bytes.toBytes(oldValue)));
560 
561     // snapshot the store.
562     this.store.snapshot();
563 
564     // add other things:
565     this.store.add(new KeyValue(row, family, qf2,
566         System.currentTimeMillis(),
567         Bytes.toBytes(oldValue)));
568 
569     // update during the snapshot.
570     long ret = this.store.updateColumnValue(row, family, qf1, newValue);
571 
572     // memstore should have grown by some amount.
573     Assert.assertTrue(ret > 0);
574 
575     // then flush.
576     flushStore(store, id++);
577     Assert.assertEquals(1, this.store.getStorefiles().size());
578     // from the one we inserted up there, and a new one
579     Assert.assertEquals(2, this.store.memstore.kvset.size());
580 
581     // how many key/values for this row are there?
582     Get get = new Get(row);
583     get.addColumn(family, qf1);
584     get.setMaxVersions(); // all versions.
585     List<Cell> results = new ArrayList<Cell>();
586 
587     results = HBaseTestingUtility.getFromStoreFile(store, get);
588     Assert.assertEquals(2, results.size());
589 
590     long ts1 = results.get(0).getTimestamp();
591     long ts2 = results.get(1).getTimestamp();
592 
593     Assert.assertTrue(ts1 > ts2);
594 
595     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
596     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
597   }
598 
599   @After
600   public void tearDown() throws Exception {
601     EnvironmentEdgeManagerTestHelper.reset();
602   }
603 
604   @Test
605   public void testICV_negMemstoreSize()  throws IOException {
606       init(this.name.getMethodName());
607 
608     long time = 100;
609     ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
610     ee.setValue(time);
611     EnvironmentEdgeManagerTestHelper.injectEdge(ee);
612     long newValue = 3L;
613     long size = 0;
614 
615 
616     size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
617         System.currentTimeMillis(),
618         Bytes.toBytes(newValue)));
619     size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
620         System.currentTimeMillis(),
621         Bytes.toBytes(newValue)));
622     size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
623         System.currentTimeMillis(),
624         Bytes.toBytes(newValue)));
625     size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
626         System.currentTimeMillis(),
627         Bytes.toBytes(newValue)));
628     size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
629         System.currentTimeMillis(),
630         Bytes.toBytes(newValue)));
631 
632 
633     for ( int i = 0 ; i < 10000 ; ++i) {
634       newValue++;
635 
636       long ret = this.store.updateColumnValue(row, family, qf1, newValue);
637       long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
638 
639       if (ret != 0) System.out.println("ret: " + ret);
640       if (ret2 != 0) System.out.println("ret2: " + ret2);
641 
642       Assert.assertTrue("ret: " + ret, ret >= 0);
643       size += ret;
644       Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
645       size += ret2;
646 
647 
648       if (i % 1000 == 0)
649         ee.setValue(++time);
650     }
651 
652     long computedSize=0;
653     for (KeyValue kv : this.store.memstore.kvset) {
654       long kvsize = MemStore.heapSizeChange(kv, true);
655       //System.out.println(kv + " size= " + kvsize + " kvsize= " + kv.heapSize());
656       computedSize += kvsize;
657     }
658     Assert.assertEquals(computedSize, size);
659   }
660 
661   @Test
662   public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
663     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
664     EnvironmentEdgeManagerTestHelper.injectEdge(mee);
665     init(this.name.getMethodName());
666 
667     long oldValue = 1L;
668     long newValue = 3L;
669     this.store.add(new KeyValue(row, family, qf1,
670         EnvironmentEdgeManager.currentTimeMillis(),
671         Bytes.toBytes(oldValue)));
672 
673     // snapshot the store.
674     this.store.snapshot();
675 
676     // update during the snapshot, the exact same TS as the Put (lololol)
677     long ret = this.store.updateColumnValue(row, family, qf1, newValue);
678 
679     // memstore should have grown by some amount.
680     Assert.assertTrue(ret > 0);
681 
682     // then flush.
683     flushStore(store, id++);
684     Assert.assertEquals(1, this.store.getStorefiles().size());
685     Assert.assertEquals(1, this.store.memstore.kvset.size());
686 
687     // now increment again:
688     newValue += 1;
689     this.store.updateColumnValue(row, family, qf1, newValue);
690 
691     // at this point we have a TS=1 in snapshot, and a TS=2 in kvset, so increment again:
692     newValue += 1;
693     this.store.updateColumnValue(row, family, qf1, newValue);
694 
695     // the second TS should be TS=2 or higher., even though 'time=1' right now.
696 
697 
698     // how many key/values for this row are there?
699     Get get = new Get(row);
700     get.addColumn(family, qf1);
701     get.setMaxVersions(); // all versions.
702     List<Cell> results = new ArrayList<Cell>();
703 
704     results = HBaseTestingUtility.getFromStoreFile(store, get);
705     Assert.assertEquals(2, results.size());
706 
707     long ts1 = results.get(0).getTimestamp();
708     long ts2 = results.get(1).getTimestamp();
709 
710     Assert.assertTrue(ts1 > ts2);
711     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
712     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
713 
714     mee.setValue(2); // time goes up slightly
715     newValue += 1;
716     this.store.updateColumnValue(row, family, qf1, newValue);
717 
718     results = HBaseTestingUtility.getFromStoreFile(store, get);
719     Assert.assertEquals(2, results.size());
720 
721     ts1 = results.get(0).getTimestamp();
722     ts2 = results.get(1).getTimestamp();
723 
724     Assert.assertTrue(ts1 > ts2);
725     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
726     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
727   }
728 
729   @Test
730   public void testHandleErrorsInFlush() throws Exception {
731     LOG.info("Setting up a faulty file system that cannot write");
732 
733     final Configuration conf = HBaseConfiguration.create();
734     User user = User.createUserForTesting(conf,
735         "testhandleerrorsinflush", new String[]{"foo"});
736     // Inject our faulty LocalFileSystem
737     conf.setClass("fs.file.impl", FaultyFileSystem.class,
738         FileSystem.class);
739     user.runAs(new PrivilegedExceptionAction<Object>() {
740       public Object run() throws Exception {
741         // Make sure it worked (above is sensitive to caching details in hadoop core)
742         FileSystem fs = FileSystem.get(conf);
743         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
744 
745         // Initialize region
746         init(name.getMethodName(), conf);
747 
748         LOG.info("Adding some data");
749         store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
750         store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
751         store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
752 
753         LOG.info("Before flush, we should have no files");
754 
755         Collection<StoreFileInfo> files =
756           store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
757         Assert.assertEquals(0, files != null ? files.size() : 0);
758 
759         //flush
760         try {
761           LOG.info("Flushing");
762           flush(1);
763           Assert.fail("Didn't bubble up IOE!");
764         } catch (IOException ioe) {
765           Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
766         }
767 
768         LOG.info("After failed flush, we should still have no files!");
769         files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
770         Assert.assertEquals(0, files != null ? files.size() : 0);
771         store.getHRegion().getLog().closeAndDelete();
772         return null;
773       }
774     });
775     FileSystem.closeAllForUGI(user.getUGI());
776   }
777 
778   /**
779    * Faulty file system that will fail if you write past its fault position the FIRST TIME
780    * only; thereafter it will succeed.  Used by {@link TestHRegion} too.
781    */
782   static class FaultyFileSystem extends FilterFileSystem {
783     List<SoftReference<FaultyOutputStream>> outStreams =
784       new ArrayList<SoftReference<FaultyOutputStream>>();
785     private long faultPos = 200;
786     AtomicBoolean fault = new AtomicBoolean(true);
787 
788     public FaultyFileSystem() {
789       super(new LocalFileSystem());
790       System.err.println("Creating faulty!");
791     }
792 
793     @Override
794     public FSDataOutputStream create(Path p) throws IOException {
795       return new FaultyOutputStream(super.create(p), faultPos, fault);
796     }
797 
798     @Override
799     public FSDataOutputStream create(Path f, FsPermission permission,
800         boolean overwrite, int bufferSize, short replication, long blockSize,
801         Progressable progress) throws IOException {
802       return new FaultyOutputStream(super.create(f, permission,
803           overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
804     }
805 
806     public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
807         int bufferSize, short replication, long blockSize, Progressable progress)
808     throws IOException {
809       // Fake it.  Call create instead.  The default implementation throws an IOE
810       // that this is not supported.
811       return create(f, overwrite, bufferSize, replication, blockSize, progress);
812     }
813   }
814 
815   static class FaultyOutputStream extends FSDataOutputStream {
816     volatile long faultPos = Long.MAX_VALUE;
817     private final AtomicBoolean fault;
818 
819     public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
820     throws IOException {
821       super(out, null);
822       this.faultPos = faultPos;
823       this.fault = fault;
824     }
825 
826     @Override
827     public void write(byte[] buf, int offset, int length) throws IOException {
828       System.err.println("faulty stream write at pos " + getPos());
829       injectFault();
830       super.write(buf, offset, length);
831     }
832 
833     private void injectFault() throws IOException {
834       if (this.fault.get() && getPos() >= faultPos) {
835         throw new IOException("Fault injected");
836       }
837     }
838   }
839 
840   private static void flushStore(HStore store, long id) throws IOException {
841     StoreFlushContext storeFlushCtx = store.createFlushContext(id);
842     storeFlushCtx.prepare();
843     storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
844     storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
845   }
846 
847   /**
848    * Generate a list of KeyValues for testing based on given parameters
849    * @param timestamps
850    * @param numRows
851    * @param qualifier
852    * @param family
853    * @return
854    */
855   List<Cell> getKeyValueSet(long[] timestamps, int numRows,
856       byte[] qualifier, byte[] family) {
857     List<Cell> kvList = new ArrayList<Cell>();
858     for (int i=1;i<=numRows;i++) {
859       byte[] b = Bytes.toBytes(i);
860       for (long timestamp: timestamps) {
861         kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
862       }
863     }
864     return kvList;
865   }
866 
867   /**
868    * Test to ensure correctness when using Stores with multiple timestamps
869    * @throws IOException
870    */
871   @Test
872   public void testMultipleTimestamps() throws IOException {
873     int numRows = 1;
874     long[] timestamps1 = new long[] {1,5,10,20};
875     long[] timestamps2 = new long[] {30,80};
876 
877     init(this.name.getMethodName());
878 
879     List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
880     for (Cell kv : kvList1) {
881       this.store.add(KeyValueUtil.ensureKeyValue(kv));
882     }
883 
884     this.store.snapshot();
885     flushStore(store, id++);
886 
887     List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
888     for(Cell kv : kvList2) {
889       this.store.add(KeyValueUtil.ensureKeyValue(kv));
890     }
891 
892     List<Cell> result;
893     Get get = new Get(Bytes.toBytes(1));
894     get.addColumn(family,qf1);
895 
896     get.setTimeRange(0,15);
897     result = HBaseTestingUtility.getFromStoreFile(store, get);
898     Assert.assertTrue(result.size()>0);
899 
900     get.setTimeRange(40,90);
901     result = HBaseTestingUtility.getFromStoreFile(store, get);
902     Assert.assertTrue(result.size()>0);
903 
904     get.setTimeRange(10,45);
905     result = HBaseTestingUtility.getFromStoreFile(store, get);
906     Assert.assertTrue(result.size()>0);
907 
908     get.setTimeRange(80,145);
909     result = HBaseTestingUtility.getFromStoreFile(store, get);
910     Assert.assertTrue(result.size()>0);
911 
912     get.setTimeRange(1,2);
913     result = HBaseTestingUtility.getFromStoreFile(store, get);
914     Assert.assertTrue(result.size()>0);
915 
916     get.setTimeRange(90,200);
917     result = HBaseTestingUtility.getFromStoreFile(store, get);
918     Assert.assertTrue(result.size()==0);
919   }
920 
921   /**
922    * Test for HBASE-3492 - Test split on empty colfam (no store files).
923    *
924    * @throws IOException When the IO operations fail.
925    */
926   @Test
927   public void testSplitWithEmptyColFam() throws IOException {
928     init(this.name.getMethodName());
929     Assert.assertNull(store.getSplitPoint());
930     store.getHRegion().forceSplit(null);
931     Assert.assertNull(store.getSplitPoint());
932     store.getHRegion().clearSplit();
933   }
934 
935   @Test
936   public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
937     final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
938     long anyValue = 10;
939 
940     // We'll check that it uses correct config and propagates it appropriately by going thru
941     // the simplest "real" path I can find - "throttleCompaction", which just checks whether
942     // a number we pass in is higher than some config value, inside compactionPolicy.
943     Configuration conf = HBaseConfiguration.create();
944     conf.setLong(CONFIG_KEY, anyValue);
945     init(name.getMethodName() + "-xml", conf);
946     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
947     Assert.assertFalse(store.throttleCompaction(anyValue));
948 
949     // HTD overrides XML.
950     --anyValue;
951     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
952     HColumnDescriptor hcd = new HColumnDescriptor(family);
953     htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
954     init(name.getMethodName() + "-htd", conf, htd, hcd);
955     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
956     Assert.assertFalse(store.throttleCompaction(anyValue));
957 
958     // HCD overrides them both.
959     --anyValue;
960     hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
961     init(name.getMethodName() + "-hcd", conf, htd, hcd);
962     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
963     Assert.assertFalse(store.throttleCompaction(anyValue));
964   }
965 
966   public static class DummyStoreEngine extends DefaultStoreEngine {
967     public static DefaultCompactor lastCreatedCompactor = null;
968     @Override
969     protected void createComponents(
970         Configuration conf, Store store, KVComparator comparator) throws IOException {
971       super.createComponents(conf, store, comparator);
972       lastCreatedCompactor = this.compactor;
973     }
974   }
975 
976   @Test
977   public void testStoreUsesSearchEngineOverride() throws Exception {
978     Configuration conf = HBaseConfiguration.create();
979     conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
980     init(this.name.getMethodName(), conf);
981     Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
982       this.store.storeEngine.getCompactor());
983   }
984 }