View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.lang.ref.SoftReference;
24  import java.security.PrivilegedExceptionAction;
25  import java.util.ArrayList;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.NavigableSet;
31  import java.util.concurrent.ConcurrentSkipListSet;
32  import java.util.concurrent.atomic.AtomicBoolean;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FSDataOutputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.FilterFileSystem;
41  import org.apache.hadoop.fs.LocalFileSystem;
42  import org.apache.hadoop.fs.Path;
43  import org.apache.hadoop.fs.permission.FsPermission;
44  import org.apache.hadoop.hbase.Cell;
45  import org.apache.hadoop.hbase.CellUtil;
46  import org.apache.hadoop.hbase.HBaseConfiguration;
47  import org.apache.hadoop.hbase.HBaseTestingUtility;
48  import org.apache.hadoop.hbase.HColumnDescriptor;
49  import org.apache.hadoop.hbase.HRegionInfo;
50  import org.apache.hadoop.hbase.HTableDescriptor;
51  import org.apache.hadoop.hbase.KeyValue;
52  import org.apache.hadoop.hbase.KeyValue.KVComparator;
53  import org.apache.hadoop.hbase.KeyValueUtil;
54  import org.apache.hadoop.hbase.MediumTests;
55  import org.apache.hadoop.hbase.TableName;
56  import org.apache.hadoop.hbase.client.Get;
57  import org.apache.hadoop.hbase.io.compress.Compression;
58  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
59  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
60  import org.apache.hadoop.hbase.io.hfile.HFile;
61  import org.apache.hadoop.hbase.io.hfile.HFileContext;
62  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
63  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
64  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
65  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
66  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
67  import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
68  import org.apache.hadoop.hbase.regionserver.wal.HLog;
69  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
70  import org.apache.hadoop.hbase.security.User;
71  import org.apache.hadoop.hbase.util.Bytes;
72  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
74  import org.apache.hadoop.hbase.util.FSUtils;
75  import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
76  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
77  import org.apache.hadoop.util.Progressable;
78  import org.junit.After;
79  import org.junit.Assert;
80  import org.junit.Before;
81  import org.junit.Rule;
82  import org.junit.Test;
83  import org.junit.experimental.categories.Category;
84  import org.junit.rules.TestName;
85  import org.mockito.Mockito;
86  
87  /**
88   * Test class for the Store
89   */
90  @Category(MediumTests.class)
91  public class TestStore {
92    public static final Log LOG = LogFactory.getLog(TestStore.class);
93    @Rule public TestName name = new TestName();
94  
95    HStore store;
96    byte [] table = Bytes.toBytes("table");
97    byte [] family = Bytes.toBytes("family");
98  
99    byte [] row = Bytes.toBytes("row");
100   byte [] row2 = Bytes.toBytes("row2");
101   byte [] qf1 = Bytes.toBytes("qf1");
102   byte [] qf2 = Bytes.toBytes("qf2");
103   byte [] qf3 = Bytes.toBytes("qf3");
104   byte [] qf4 = Bytes.toBytes("qf4");
105   byte [] qf5 = Bytes.toBytes("qf5");
106   byte [] qf6 = Bytes.toBytes("qf6");
107 
108   NavigableSet<byte[]> qualifiers =
109     new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
110 
111   List<Cell> expected = new ArrayList<Cell>();
112   List<Cell> result = new ArrayList<Cell>();
113 
114   long id = System.currentTimeMillis();
115   Get get = new Get(row);
116 
117   private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
118   private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
119 
120 
121   /**
122    * Setup
123    * @throws IOException
124    */
125   @Before
126   public void setUp() throws IOException {
127     qualifiers.add(qf1);
128     qualifiers.add(qf3);
129     qualifiers.add(qf5);
130 
131     Iterator<byte[]> iter = qualifiers.iterator();
132     while(iter.hasNext()){
133       byte [] next = iter.next();
134       expected.add(new KeyValue(row, family, next, 1, (byte[])null));
135       get.addColumn(family, next);
136     }
137   }
138 
139   private void init(String methodName) throws IOException {
140     init(methodName, HBaseConfiguration.create());
141   }
142 
143   private void init(String methodName, Configuration conf)
144   throws IOException {
145     HColumnDescriptor hcd = new HColumnDescriptor(family);
146     // some of the tests write 4 versions and then flush
147     // (with HBASE-4241, lower versions are collected on flush)
148     hcd.setMaxVersions(4);
149     init(methodName, conf, hcd);
150   }
151 
152   private void init(String methodName, Configuration conf,
153       HColumnDescriptor hcd) throws IOException {
154     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
155     init(methodName, conf, htd, hcd);
156   }
157 
158   @SuppressWarnings("deprecation")
159   private Store init(String methodName, Configuration conf, HTableDescriptor htd,
160       HColumnDescriptor hcd) throws IOException {
161     //Setting up a Store
162     Path basedir = new Path(DIR+methodName);
163     Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
164     String logName = "logs";
165     Path logdir = new Path(basedir, logName);
166 
167     FileSystem fs = FileSystem.get(conf);
168 
169     fs.delete(logdir, true);
170 
171     htd.addFamily(hcd);
172     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
173     HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
174     HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
175 
176     store = new HStore(region, hcd, conf);
177     return store;
178   }
179 
180   /**
181    * Test we do not lose data if we fail a flush and then close.
182    * Part of HBase-10466
183    * @throws Exception
184    */
185   @Test
186   public void testFlushSizeAccounting() throws Exception {
187     LOG.info("Setting up a faulty file system that cannot write in " +
188       this.name.getMethodName());
189     final Configuration conf = HBaseConfiguration.create();
190     // Only retry once.
191     conf.setInt("hbase.hstore.flush.retries.number", 1);
192     User user = User.createUserForTesting(conf, this.name.getMethodName(),
193       new String[]{"foo"});
194     // Inject our faulty LocalFileSystem
195     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
196     user.runAs(new PrivilegedExceptionAction<Object>() {
197       public Object run() throws Exception {
198         // Make sure it worked (above is sensitive to caching details in hadoop core)
199         FileSystem fs = FileSystem.get(conf);
200         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
201         FaultyFileSystem ffs = (FaultyFileSystem)fs;
202 
203         // Initialize region
204         init(name.getMethodName(), conf);
205 
206         long size = store.memstore.getFlushableSize();
207         Assert.assertEquals(0, size);
208         LOG.info("Adding some data");
209         long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
210         size = store.memstore.getFlushableSize();
211         Assert.assertEquals(kvSize, size);
212         // Flush.  Bug #1 from HBASE-10466.  Make sure size calculation on failed flush is right.
213         try {
214           LOG.info("Flushing");
215           flushStore(store, id++);
216           Assert.fail("Didn't bubble up IOE!");
217         } catch (IOException ioe) {
218           Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
219         }
220         size = store.memstore.getFlushableSize();
221         Assert.assertEquals(kvSize, size);
222         store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
223         // Even though we add a new kv, we expect the flushable size to be 'same' since we have
224         // not yet cleared the snapshot -- the above flush failed.
225         Assert.assertEquals(kvSize, size);
226         ffs.fault.set(false);
227         flushStore(store, id++);
228         size = store.memstore.getFlushableSize();
229         // Size should be the foreground kv size.
230         Assert.assertEquals(kvSize, size);
231         flushStore(store, id++);
232         size = store.memstore.getFlushableSize();
233         Assert.assertEquals(0, size);
234         return null;
235       }
236     });
237   }
238 
239   /**
240    * Verify that compression and data block encoding are respected by the
241    * Store.createWriterInTmp() method, used on store flush.
242    */
243   @Test
244   public void testCreateWriter() throws Exception {
245     Configuration conf = HBaseConfiguration.create();
246     FileSystem fs = FileSystem.get(conf);
247 
248     HColumnDescriptor hcd = new HColumnDescriptor(family);
249     hcd.setCompressionType(Compression.Algorithm.GZ);
250     hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
251     init(name.getMethodName(), conf, hcd);
252 
253     // Test createWriterInTmp()
254     StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
255     Path path = writer.getPath();
256     writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
257     writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
258     writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
259     writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
260     writer.close();
261 
262     // Verify that compression and encoding settings are respected
263     HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
264     Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
265     Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
266     reader.close();
267   }
268 
269   @Test
270   public void testDeleteExpiredStoreFiles() throws Exception {
271     int storeFileNum = 4;
272     int ttl = 4;
273     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
274     EnvironmentEdgeManagerTestHelper.injectEdge(edge);
275     
276     Configuration conf = HBaseConfiguration.create();
277     // Enable the expired store file deletion
278     conf.setBoolean("hbase.store.delete.expired.storefile", true);
279     // Set the compaction threshold higher to avoid normal compactions.
280     conf.setInt(CompactionConfiguration.MIN_KEY, 5);
281 
282     HColumnDescriptor hcd = new HColumnDescriptor(family);
283     hcd.setTimeToLive(ttl);
284     init(name.getMethodName(), conf, hcd);
285 
286     long storeTtl = this.store.getScanInfo().getTtl();
287     long sleepTime = storeTtl / storeFileNum;
288     long timeStamp;
289     // There are 4 store files and the max time stamp difference among these
290     // store files will be (this.store.ttl / storeFileNum)
291     for (int i = 1; i <= storeFileNum; i++) {
292       LOG.info("Adding some data for the store file #" + i);
293       timeStamp = EnvironmentEdgeManager.currentTimeMillis();
294       this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
295       this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
296       this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
297       flush(i);
298       edge.incrementTime(sleepTime);
299     }
300 
301     // Verify the total number of store files
302     Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
303 
304      // Each call will find one expired store file and delete it before compaction happens.
305      // There will be no compaction due to threshold above. Last file will not be replaced.
306     for (int i = 1; i <= storeFileNum - 1; i++) {
307       // verify the expired store file.
308       Assert.assertNull(this.store.requestCompaction());
309       Collection<StoreFile> sfs = this.store.getStorefiles();
310       // Ensure i files are gone.
311       Assert.assertEquals(storeFileNum - i, sfs.size());
312       // Ensure only non-expired files remain.
313       for (StoreFile sf : sfs) {
314         Assert.assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTimeMillis() - storeTtl));
315       }
316       // Let the next store file expired.
317       edge.incrementTime(sleepTime);
318     }
319     Assert.assertNull(this.store.requestCompaction());
320     Collection<StoreFile> sfs = this.store.getStorefiles();
321     // Assert the last expired file is not removed.
322     Assert.assertEquals(1, sfs.size());
323     long ts = sfs.iterator().next().getReader().getMaxTimestamp();
324     Assert.assertTrue(ts < (edge.currentTimeMillis() - storeTtl));
325   }
326 
327   @Test
328   public void testLowestModificationTime() throws Exception {
329     Configuration conf = HBaseConfiguration.create();
330     FileSystem fs = FileSystem.get(conf);
331     // Initialize region
332     init(name.getMethodName(), conf);
333     
334     int storeFileNum = 4;
335     for (int i = 1; i <= storeFileNum; i++) {
336       LOG.info("Adding some data for the store file #"+i);
337       this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
338       this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
339       this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
340       flush(i);
341     }
342     // after flush; check the lowest time stamp
343     long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
344     long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
345     Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
346 
347     // after compact; check the lowest time stamp
348     store.compact(store.requestCompaction());
349     lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
350     lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
351     Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
352   }
353   
354   private static long getLowestTimeStampFromFS(FileSystem fs, 
355       final Collection<StoreFile> candidates) throws IOException {
356     long minTs = Long.MAX_VALUE;
357     if (candidates.isEmpty()) {
358       return minTs; 
359     }
360     Path[] p = new Path[candidates.size()];
361     int i = 0;
362     for (StoreFile sf : candidates) {
363       p[i] = sf.getPath();
364       ++i;
365     }
366     
367     FileStatus[] stats = fs.listStatus(p);
368     if (stats == null || stats.length == 0) {
369       return minTs;
370     }
371     for (FileStatus s : stats) {
372       minTs = Math.min(minTs, s.getModificationTime());
373     }
374     return minTs;
375   }
376 
377   //////////////////////////////////////////////////////////////////////////////
378   // Get tests
379   //////////////////////////////////////////////////////////////////////////////
380 
381   private static final int BLOCKSIZE_SMALL = 8192;
382   /**
383    * Test for hbase-1686.
384    * @throws IOException
385    */
386   @Test
387   public void testEmptyStoreFile() throws IOException {
388     init(this.name.getMethodName());
389     // Write a store file.
390     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
391     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
392     flush(1);
393     // Now put in place an empty store file.  Its a little tricky.  Have to
394     // do manually with hacked in sequence id.
395     StoreFile f = this.store.getStorefiles().iterator().next();
396     Path storedir = f.getPath().getParent();
397     long seqid = f.getMaxSequenceId();
398     Configuration c = HBaseConfiguration.create();
399     FileSystem fs = FileSystem.get(c);
400     HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
401     StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
402         fs)
403             .withOutputDir(storedir)
404             .withFileContext(meta)
405             .build();
406     w.appendMetadata(seqid + 1, false);
407     w.close();
408     this.store.close();
409     // Reopen it... should pick up two files
410     this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
411     Assert.assertEquals(2, this.store.getStorefilesCount());
412 
413     result = HBaseTestingUtility.getFromStoreFile(store,
414         get.getRow(),
415         qualifiers);
416     Assert.assertEquals(1, result.size());
417   }
418 
419   /**
420    * Getting data from memstore only
421    * @throws IOException
422    */
423   @Test
424   public void testGet_FromMemStoreOnly() throws IOException {
425     init(this.name.getMethodName());
426 
427     //Put data in memstore
428     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
429     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
430     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
431     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
432     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
433     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
434 
435     //Get
436     result = HBaseTestingUtility.getFromStoreFile(store,
437         get.getRow(), qualifiers);
438 
439     //Compare
440     assertCheck();
441   }
442 
443   /**
444    * Getting data from files only
445    * @throws IOException
446    */
447   @Test
448   public void testGet_FromFilesOnly() throws IOException {
449     init(this.name.getMethodName());
450 
451     //Put data in memstore
452     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
453     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
454     //flush
455     flush(1);
456 
457     //Add more data
458     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
459     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
460     //flush
461     flush(2);
462 
463     //Add more data
464     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
465     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
466     //flush
467     flush(3);
468 
469     //Get
470     result = HBaseTestingUtility.getFromStoreFile(store,
471         get.getRow(),
472         qualifiers);
473     //this.store.get(get, qualifiers, result);
474 
475     //Need to sort the result since multiple files
476     Collections.sort(result, KeyValue.COMPARATOR);
477 
478     //Compare
479     assertCheck();
480   }
481 
482   /**
483    * Getting data from memstore and files
484    * @throws IOException
485    */
486   @Test
487   public void testGet_FromMemStoreAndFiles() throws IOException {
488     init(this.name.getMethodName());
489 
490     //Put data in memstore
491     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
492     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
493     //flush
494     flush(1);
495 
496     //Add more data
497     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
498     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
499     //flush
500     flush(2);
501 
502     //Add more data
503     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
504     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
505 
506     //Get
507     result = HBaseTestingUtility.getFromStoreFile(store,
508         get.getRow(), qualifiers);
509 
510     //Need to sort the result since multiple files
511     Collections.sort(result, KeyValue.COMPARATOR);
512 
513     //Compare
514     assertCheck();
515   }
516 
517   private void flush(int storeFilessize) throws IOException{
518     this.store.snapshot();
519     flushStore(store, id++);
520     Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
521     Assert.assertEquals(0, this.store.memstore.kvset.size());
522   }
523 
524   private void assertCheck() {
525     Assert.assertEquals(expected.size(), result.size());
526     for(int i=0; i<expected.size(); i++) {
527       Assert.assertEquals(expected.get(i), result.get(i));
528     }
529   }
530 
531   //////////////////////////////////////////////////////////////////////////////
532   // IncrementColumnValue tests
533   //////////////////////////////////////////////////////////////////////////////
534   /*
535    * test the internal details of how ICV works, especially during a flush scenario.
536    */
537   @Test
538   public void testIncrementColumnValue_ICVDuringFlush()
539       throws IOException, InterruptedException {
540     init(this.name.getMethodName());
541 
542     long oldValue = 1L;
543     long newValue = 3L;
544     this.store.add(new KeyValue(row, family, qf1,
545         System.currentTimeMillis(),
546         Bytes.toBytes(oldValue)));
547 
548     // snapshot the store.
549     this.store.snapshot();
550 
551     // add other things:
552     this.store.add(new KeyValue(row, family, qf2,
553         System.currentTimeMillis(),
554         Bytes.toBytes(oldValue)));
555 
556     // update during the snapshot.
557     long ret = this.store.updateColumnValue(row, family, qf1, newValue);
558 
559     // memstore should have grown by some amount.
560     Assert.assertTrue(ret > 0);
561 
562     // then flush.
563     flushStore(store, id++);
564     Assert.assertEquals(1, this.store.getStorefiles().size());
565     // from the one we inserted up there, and a new one
566     Assert.assertEquals(2, this.store.memstore.kvset.size());
567 
568     // how many key/values for this row are there?
569     Get get = new Get(row);
570     get.addColumn(family, qf1);
571     get.setMaxVersions(); // all versions.
572     List<Cell> results = new ArrayList<Cell>();
573 
574     results = HBaseTestingUtility.getFromStoreFile(store, get);
575     Assert.assertEquals(2, results.size());
576 
577     long ts1 = results.get(0).getTimestamp();
578     long ts2 = results.get(1).getTimestamp();
579 
580     Assert.assertTrue(ts1 > ts2);
581 
582     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
583     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
584   }
585 
586   @After
587   public void tearDown() throws Exception {
588     EnvironmentEdgeManagerTestHelper.reset();
589   }
590 
591   @Test
592   public void testICV_negMemstoreSize()  throws IOException {
593       init(this.name.getMethodName());
594 
595     long time = 100;
596     ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
597     ee.setValue(time);
598     EnvironmentEdgeManagerTestHelper.injectEdge(ee);
599     long newValue = 3L;
600     long size = 0;
601 
602 
603     size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
604         System.currentTimeMillis(),
605         Bytes.toBytes(newValue)));
606     size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
607         System.currentTimeMillis(),
608         Bytes.toBytes(newValue)));
609     size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
610         System.currentTimeMillis(),
611         Bytes.toBytes(newValue)));
612     size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
613         System.currentTimeMillis(),
614         Bytes.toBytes(newValue)));
615     size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
616         System.currentTimeMillis(),
617         Bytes.toBytes(newValue)));
618 
619 
620     for ( int i = 0 ; i < 10000 ; ++i) {
621       newValue++;
622 
623       long ret = this.store.updateColumnValue(row, family, qf1, newValue);
624       long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
625 
626       if (ret != 0) System.out.println("ret: " + ret);
627       if (ret2 != 0) System.out.println("ret2: " + ret2);
628 
629       Assert.assertTrue("ret: " + ret, ret >= 0);
630       size += ret;
631       Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
632       size += ret2;
633 
634 
635       if (i % 1000 == 0)
636         ee.setValue(++time);
637     }
638 
639     long computedSize=0;
640     for (KeyValue kv : this.store.memstore.kvset) {
641       long kvsize = MemStore.heapSizeChange(kv, true);
642       //System.out.println(kv + " size= " + kvsize + " kvsize= " + kv.heapSize());
643       computedSize += kvsize;
644     }
645     Assert.assertEquals(computedSize, size);
646   }
647 
648   @Test
649   public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
650     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
651     EnvironmentEdgeManagerTestHelper.injectEdge(mee);
652     init(this.name.getMethodName());
653 
654     long oldValue = 1L;
655     long newValue = 3L;
656     this.store.add(new KeyValue(row, family, qf1,
657         EnvironmentEdgeManager.currentTimeMillis(),
658         Bytes.toBytes(oldValue)));
659 
660     // snapshot the store.
661     this.store.snapshot();
662 
663     // update during the snapshot, the exact same TS as the Put (lololol)
664     long ret = this.store.updateColumnValue(row, family, qf1, newValue);
665 
666     // memstore should have grown by some amount.
667     Assert.assertTrue(ret > 0);
668 
669     // then flush.
670     flushStore(store, id++);
671     Assert.assertEquals(1, this.store.getStorefiles().size());
672     Assert.assertEquals(1, this.store.memstore.kvset.size());
673 
674     // now increment again:
675     newValue += 1;
676     this.store.updateColumnValue(row, family, qf1, newValue);
677 
678     // at this point we have a TS=1 in snapshot, and a TS=2 in kvset, so increment again:
679     newValue += 1;
680     this.store.updateColumnValue(row, family, qf1, newValue);
681 
682     // the second TS should be TS=2 or higher., even though 'time=1' right now.
683 
684 
685     // how many key/values for this row are there?
686     Get get = new Get(row);
687     get.addColumn(family, qf1);
688     get.setMaxVersions(); // all versions.
689     List<Cell> results = new ArrayList<Cell>();
690 
691     results = HBaseTestingUtility.getFromStoreFile(store, get);
692     Assert.assertEquals(2, results.size());
693 
694     long ts1 = results.get(0).getTimestamp();
695     long ts2 = results.get(1).getTimestamp();
696 
697     Assert.assertTrue(ts1 > ts2);
698     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
699     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
700 
701     mee.setValue(2); // time goes up slightly
702     newValue += 1;
703     this.store.updateColumnValue(row, family, qf1, newValue);
704 
705     results = HBaseTestingUtility.getFromStoreFile(store, get);
706     Assert.assertEquals(2, results.size());
707 
708     ts1 = results.get(0).getTimestamp();
709     ts2 = results.get(1).getTimestamp();
710 
711     Assert.assertTrue(ts1 > ts2);
712     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
713     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
714   }
715 
716   @Test
717   public void testHandleErrorsInFlush() throws Exception {
718     LOG.info("Setting up a faulty file system that cannot write");
719 
720     final Configuration conf = HBaseConfiguration.create();
721     User user = User.createUserForTesting(conf,
722         "testhandleerrorsinflush", new String[]{"foo"});
723     // Inject our faulty LocalFileSystem
724     conf.setClass("fs.file.impl", FaultyFileSystem.class,
725         FileSystem.class);
726     user.runAs(new PrivilegedExceptionAction<Object>() {
727       public Object run() throws Exception {
728         // Make sure it worked (above is sensitive to caching details in hadoop core)
729         FileSystem fs = FileSystem.get(conf);
730         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
731 
732         // Initialize region
733         init(name.getMethodName(), conf);
734 
735         LOG.info("Adding some data");
736         store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
737         store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
738         store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
739 
740         LOG.info("Before flush, we should have no files");
741 
742         Collection<StoreFileInfo> files =
743           store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
744         Assert.assertEquals(0, files != null ? files.size() : 0);
745 
746         //flush
747         try {
748           LOG.info("Flushing");
749           flush(1);
750           Assert.fail("Didn't bubble up IOE!");
751         } catch (IOException ioe) {
752           Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
753         }
754 
755         LOG.info("After failed flush, we should still have no files!");
756         files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
757         Assert.assertEquals(0, files != null ? files.size() : 0);
758         store.getHRegion().getLog().closeAndDelete();
759         return null;
760       }
761     });
762     FileSystem.closeAllForUGI(user.getUGI());
763   }
764 
765   /**
766    * Faulty file system that will fail if you write past its fault position the FIRST TIME
767    * only; thereafter it will succeed.  Used by {@link TestHRegion} too.
768    */
769   static class FaultyFileSystem extends FilterFileSystem {
770     List<SoftReference<FaultyOutputStream>> outStreams =
771       new ArrayList<SoftReference<FaultyOutputStream>>();
772     private long faultPos = 200;
773     AtomicBoolean fault = new AtomicBoolean(true);
774 
775     public FaultyFileSystem() {
776       super(new LocalFileSystem());
777       System.err.println("Creating faulty!");
778     }
779 
780     @Override
781     public FSDataOutputStream create(Path p) throws IOException {
782       return new FaultyOutputStream(super.create(p), faultPos, fault);
783     }
784 
785     @Override
786     public FSDataOutputStream create(Path f, FsPermission permission,
787         boolean overwrite, int bufferSize, short replication, long blockSize,
788         Progressable progress) throws IOException {
789       return new FaultyOutputStream(super.create(f, permission,
790           overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
791     }
792 
793     public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
794         int bufferSize, short replication, long blockSize, Progressable progress)
795     throws IOException {
796       // Fake it.  Call create instead.  The default implementation throws an IOE
797       // that this is not supported.
798       return create(f, overwrite, bufferSize, replication, blockSize, progress);
799     }
800   }
801 
802   static class FaultyOutputStream extends FSDataOutputStream {
803     volatile long faultPos = Long.MAX_VALUE;
804     private final AtomicBoolean fault;
805 
806     public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
807     throws IOException {
808       super(out, null);
809       this.faultPos = faultPos;
810       this.fault = fault;
811     }
812 
813     @Override
814     public void write(byte[] buf, int offset, int length) throws IOException {
815       System.err.println("faulty stream write at pos " + getPos());
816       injectFault();
817       super.write(buf, offset, length);
818     }
819 
820     private void injectFault() throws IOException {
821       if (this.fault.get() && getPos() >= faultPos) {
822         throw new IOException("Fault injected");
823       }
824     }
825   }
826 
827   private static void flushStore(HStore store, long id) throws IOException {
828     StoreFlushContext storeFlushCtx = store.createFlushContext(id);
829     storeFlushCtx.prepare();
830     storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
831     storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
832   }
833 
834   /**
835    * Generate a list of KeyValues for testing based on given parameters
836    * @param timestamps
837    * @param numRows
838    * @param qualifier
839    * @param family
840    * @return
841    */
842   List<Cell> getKeyValueSet(long[] timestamps, int numRows,
843       byte[] qualifier, byte[] family) {
844     List<Cell> kvList = new ArrayList<Cell>();
845     for (int i=1;i<=numRows;i++) {
846       byte[] b = Bytes.toBytes(i);
847       for (long timestamp: timestamps) {
848         kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
849       }
850     }
851     return kvList;
852   }
853 
854   /**
855    * Test to ensure correctness when using Stores with multiple timestamps
856    * @throws IOException
857    */
858   @Test
859   public void testMultipleTimestamps() throws IOException {
860     int numRows = 1;
861     long[] timestamps1 = new long[] {1,5,10,20};
862     long[] timestamps2 = new long[] {30,80};
863 
864     init(this.name.getMethodName());
865 
866     List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
867     for (Cell kv : kvList1) {
868       this.store.add(KeyValueUtil.ensureKeyValue(kv));
869     }
870 
871     this.store.snapshot();
872     flushStore(store, id++);
873 
874     List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
875     for(Cell kv : kvList2) {
876       this.store.add(KeyValueUtil.ensureKeyValue(kv));
877     }
878 
879     List<Cell> result;
880     Get get = new Get(Bytes.toBytes(1));
881     get.addColumn(family,qf1);
882 
883     get.setTimeRange(0,15);
884     result = HBaseTestingUtility.getFromStoreFile(store, get);
885     Assert.assertTrue(result.size()>0);
886 
887     get.setTimeRange(40,90);
888     result = HBaseTestingUtility.getFromStoreFile(store, get);
889     Assert.assertTrue(result.size()>0);
890 
891     get.setTimeRange(10,45);
892     result = HBaseTestingUtility.getFromStoreFile(store, get);
893     Assert.assertTrue(result.size()>0);
894 
895     get.setTimeRange(80,145);
896     result = HBaseTestingUtility.getFromStoreFile(store, get);
897     Assert.assertTrue(result.size()>0);
898 
899     get.setTimeRange(1,2);
900     result = HBaseTestingUtility.getFromStoreFile(store, get);
901     Assert.assertTrue(result.size()>0);
902 
903     get.setTimeRange(90,200);
904     result = HBaseTestingUtility.getFromStoreFile(store, get);
905     Assert.assertTrue(result.size()==0);
906   }
907 
908   /**
909    * Test for HBASE-3492 - Test split on empty colfam (no store files).
910    *
911    * @throws IOException When the IO operations fail.
912    */
913   @Test
914   public void testSplitWithEmptyColFam() throws IOException {
915     init(this.name.getMethodName());
916     Assert.assertNull(store.getSplitPoint());
917     store.getHRegion().forceSplit(null);
918     Assert.assertNull(store.getSplitPoint());
919     store.getHRegion().clearSplit();
920   }
921 
922   @Test
923   public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
924     final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
925     long anyValue = 10;
926 
927     // We'll check that it uses correct config and propagates it appropriately by going thru
928     // the simplest "real" path I can find - "throttleCompaction", which just checks whether
929     // a number we pass in is higher than some config value, inside compactionPolicy.
930     Configuration conf = HBaseConfiguration.create();
931     conf.setLong(CONFIG_KEY, anyValue);
932     init(name.getMethodName() + "-xml", conf);
933     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
934     Assert.assertFalse(store.throttleCompaction(anyValue));
935 
936     // HTD overrides XML.
937     --anyValue;
938     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
939     HColumnDescriptor hcd = new HColumnDescriptor(family);
940     htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
941     init(name.getMethodName() + "-htd", conf, htd, hcd);
942     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
943     Assert.assertFalse(store.throttleCompaction(anyValue));
944 
945     // HCD overrides them both.
946     --anyValue;
947     hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
948     init(name.getMethodName() + "-hcd", conf, htd, hcd);
949     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
950     Assert.assertFalse(store.throttleCompaction(anyValue));
951   }
952 
953   public static class DummyStoreEngine extends DefaultStoreEngine {
954     public static DefaultCompactor lastCreatedCompactor = null;
955     @Override
956     protected void createComponents(
957         Configuration conf, Store store, KVComparator comparator) throws IOException {
958       super.createComponents(conf, store, comparator);
959       lastCreatedCompactor = this.compactor;
960     }
961   }
962 
963   @Test
964   public void testStoreUsesSearchEngineOverride() throws Exception {
965     Configuration conf = HBaseConfiguration.create();
966     conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
967     init(this.name.getMethodName(), conf);
968     Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
969       this.store.storeEngine.getCompactor());
970   }
971 }