View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.lang.ref.SoftReference;
24  import java.security.PrivilegedExceptionAction;
25  import java.util.ArrayList;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.NavigableSet;
31  import java.util.concurrent.ConcurrentSkipListSet;
32  import java.util.concurrent.atomic.AtomicBoolean;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FSDataOutputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.FilterFileSystem;
41  import org.apache.hadoop.fs.LocalFileSystem;
42  import org.apache.hadoop.fs.Path;
43  import org.apache.hadoop.fs.permission.FsPermission;
44  import org.apache.hadoop.hbase.Cell;
45  import org.apache.hadoop.hbase.CellUtil;
46  import org.apache.hadoop.hbase.HBaseConfiguration;
47  import org.apache.hadoop.hbase.HBaseTestingUtility;
48  import org.apache.hadoop.hbase.HColumnDescriptor;
49  import org.apache.hadoop.hbase.HRegionInfo;
50  import org.apache.hadoop.hbase.HTableDescriptor;
51  import org.apache.hadoop.hbase.KeyValue;
52  import org.apache.hadoop.hbase.KeyValue.KVComparator;
53  import org.apache.hadoop.hbase.KeyValueUtil;
54  import org.apache.hadoop.hbase.MediumTests;
55  import org.apache.hadoop.hbase.TableName;
56  import org.apache.hadoop.hbase.client.Get;
57  import org.apache.hadoop.hbase.io.compress.Compression;
58  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
59  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
60  import org.apache.hadoop.hbase.io.hfile.HFile;
61  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
62  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
63  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
64  import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
65  import org.apache.hadoop.hbase.regionserver.wal.HLog;
66  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
67  import org.apache.hadoop.hbase.security.User;
68  import org.apache.hadoop.hbase.util.Bytes;
69  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
70  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
71  import org.apache.hadoop.hbase.util.FSUtils;
72  import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
73  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
74  import org.apache.hadoop.util.Progressable;
75  import org.junit.After;
76  import org.junit.Assert;
77  import org.junit.Before;
78  import org.junit.Rule;
79  import org.junit.Test;
80  import org.junit.experimental.categories.Category;
81  import org.junit.rules.TestName;
82  import org.mockito.Mockito;
83  
84  /**
85   * Test class for the Store
86   */
87  @Category(MediumTests.class)
88  public class TestStore {
89    public static final Log LOG = LogFactory.getLog(TestStore.class);
90    @Rule public TestName name = new TestName();
91  
92    HStore store;
93    byte [] table = Bytes.toBytes("table");
94    byte [] family = Bytes.toBytes("family");
95  
96    byte [] row = Bytes.toBytes("row");
97    byte [] row2 = Bytes.toBytes("row2");
98    byte [] qf1 = Bytes.toBytes("qf1");
99    byte [] qf2 = Bytes.toBytes("qf2");
100   byte [] qf3 = Bytes.toBytes("qf3");
101   byte [] qf4 = Bytes.toBytes("qf4");
102   byte [] qf5 = Bytes.toBytes("qf5");
103   byte [] qf6 = Bytes.toBytes("qf6");
104 
105   NavigableSet<byte[]> qualifiers =
106     new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
107 
108   List<Cell> expected = new ArrayList<Cell>();
109   List<Cell> result = new ArrayList<Cell>();
110 
111   long id = System.currentTimeMillis();
112   Get get = new Get(row);
113 
114   private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
115   private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
116 
117 
118   /**
119    * Setup
120    * @throws IOException
121    */
122   @Before
123   public void setUp() throws IOException {
124     qualifiers.add(qf1);
125     qualifiers.add(qf3);
126     qualifiers.add(qf5);
127 
128     Iterator<byte[]> iter = qualifiers.iterator();
129     while(iter.hasNext()){
130       byte [] next = iter.next();
131       expected.add(new KeyValue(row, family, next, 1, (byte[])null));
132       get.addColumn(family, next);
133     }
134   }
135 
136   private void init(String methodName) throws IOException {
137     init(methodName, HBaseConfiguration.create());
138   }
139 
140   private void init(String methodName, Configuration conf)
141   throws IOException {
142     HColumnDescriptor hcd = new HColumnDescriptor(family);
143     // some of the tests write 4 versions and then flush
144     // (with HBASE-4241, lower versions are collected on flush)
145     hcd.setMaxVersions(4);
146     init(methodName, conf, hcd);
147   }
148 
149   private void init(String methodName, Configuration conf,
150       HColumnDescriptor hcd) throws IOException {
151     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
152     init(methodName, conf, htd, hcd);
153   }
154 
155   @SuppressWarnings("deprecation")
156   private Store init(String methodName, Configuration conf, HTableDescriptor htd,
157       HColumnDescriptor hcd) throws IOException {
158     //Setting up a Store
159     Path basedir = new Path(DIR+methodName);
160     Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
161     String logName = "logs";
162     Path logdir = new Path(basedir, logName);
163 
164     FileSystem fs = FileSystem.get(conf);
165 
166     fs.delete(logdir, true);
167 
168     htd.addFamily(hcd);
169     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
170     HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
171     HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
172 
173     store = new HStore(region, hcd, conf);
174     return store;
175   }
176 
177   /**
178    * Test we do not lose data if we fail a flush and then close.
179    * Part of HBase-10466
180    * @throws Exception
181    */
182   @Test
183   public void testFlushSizeAccounting() throws Exception {
184     LOG.info("Setting up a faulty file system that cannot write in " +
185       this.name.getMethodName());
186     final Configuration conf = HBaseConfiguration.create();
187     // Only retry once.
188     conf.setInt("hbase.hstore.flush.retries.number", 1);
189     User user = User.createUserForTesting(conf, this.name.getMethodName(),
190       new String[]{"foo"});
191     // Inject our faulty LocalFileSystem
192     conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
193     user.runAs(new PrivilegedExceptionAction<Object>() {
194       public Object run() throws Exception {
195         // Make sure it worked (above is sensitive to caching details in hadoop core)
196         FileSystem fs = FileSystem.get(conf);
197         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
198         FaultyFileSystem ffs = (FaultyFileSystem)fs;
199 
200         // Initialize region
201         init(name.getMethodName(), conf);
202 
203         long size = store.memstore.getFlushableSize();
204         Assert.assertEquals(0, size);
205         LOG.info("Adding some data");
206         long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
207         size = store.memstore.getFlushableSize();
208         Assert.assertEquals(kvSize, size);
209         // Flush.  Bug #1 from HBASE-10466.  Make sure size calculation on failed flush is right.
210         try {
211           LOG.info("Flushing");
212           flushStore(store, id++);
213           Assert.fail("Didn't bubble up IOE!");
214         } catch (IOException ioe) {
215           Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
216         }
217         size = store.memstore.getFlushableSize();
218         Assert.assertEquals(kvSize, size);
219         store.add(new KeyValue(row, family, qf2, 2, (byte[])null));
220         // Even though we add a new kv, we expect the flushable size to be 'same' since we have
221         // not yet cleared the snapshot -- the above flush failed.
222         Assert.assertEquals(kvSize, size);
223         ffs.fault.set(false);
224         flushStore(store, id++);
225         size = store.memstore.getFlushableSize();
226         // Size should be the foreground kv size.
227         Assert.assertEquals(kvSize, size);
228         flushStore(store, id++);
229         size = store.memstore.getFlushableSize();
230         Assert.assertEquals(0, size);
231         return null;
232       }
233     });
234   }
235 
236   /**
237    * Verify that compression and data block encoding are respected by the
238    * Store.createWriterInTmp() method, used on store flush.
239    */
240   @Test
241   public void testCreateWriter() throws Exception {
242     Configuration conf = HBaseConfiguration.create();
243     FileSystem fs = FileSystem.get(conf);
244 
245     HColumnDescriptor hcd = new HColumnDescriptor(family);
246     hcd.setCompressionType(Compression.Algorithm.GZ);
247     hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
248     init(name.getMethodName(), conf, hcd);
249 
250     // Test createWriterInTmp()
251     StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true);
252     Path path = writer.getPath();
253     writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
254     writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
255     writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
256     writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
257     writer.close();
258 
259     // Verify that compression and encoding settings are respected
260     HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf));
261     Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
262     Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
263     reader.close();
264   }
265 
266   @Test
267   public void testDeleteExpiredStoreFiles() throws Exception {
268     int storeFileNum = 4;
269     int ttl = 4;
270     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
271     EnvironmentEdgeManagerTestHelper.injectEdge(edge);
272     
273     Configuration conf = HBaseConfiguration.create();
274     // Enable the expired store file deletion
275     conf.setBoolean("hbase.store.delete.expired.storefile", true);
276     HColumnDescriptor hcd = new HColumnDescriptor(family);
277     hcd.setTimeToLive(ttl);
278     init(name.getMethodName(), conf, hcd);
279 
280     long sleepTime = this.store.getScanInfo().getTtl() / storeFileNum;
281     long timeStamp;
282     // There are 4 store files and the max time stamp difference among these
283     // store files will be (this.store.ttl / storeFileNum)
284     for (int i = 1; i <= storeFileNum; i++) {
285       LOG.info("Adding some data for the store file #" + i);
286       timeStamp = EnvironmentEdgeManager.currentTimeMillis();
287       this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
288       this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
289       this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
290       flush(i);
291       edge.incrementTime(sleepTime);
292     }
293 
294     // Verify the total number of store files
295     Assert.assertEquals(storeFileNum, this.store.getStorefiles().size());
296 
297     // Each compaction request will find one expired store file and delete it
298     // by the compaction.
299     for (int i = 1; i <= storeFileNum; i++) {
300       // verify the expired store file.
301       CompactionContext compaction = this.store.requestCompaction();
302       CompactionRequest cr = compaction.getRequest();
303       // the first is expired normally.
304       // If not the first compaction, there is another empty store file,
305       List<StoreFile> files = new ArrayList<StoreFile>(cr.getFiles());
306       Assert.assertEquals(Math.min(i, 2), cr.getFiles().size());
307       for (int j = 0; j < files.size(); j++) {
308         Assert.assertTrue(files.get(j).getReader().getMaxTimestamp() < (edge
309             .currentTimeMillis() - this.store.getScanInfo().getTtl()));
310       }
311       // Verify that the expired store file is compacted to an empty store file.
312       // Default compaction policy creates just one and only one compacted file.
313       StoreFile compactedFile = this.store.compact(compaction).get(0);
314       // It is an empty store file.
315       Assert.assertEquals(0, compactedFile.getReader().getEntries());
316 
317       // Let the next store file expired.
318       edge.incrementTime(sleepTime);
319     }
320   }
321 
322   @Test
323   public void testLowestModificationTime() throws Exception {
324     Configuration conf = HBaseConfiguration.create();
325     FileSystem fs = FileSystem.get(conf);
326     // Initialize region
327     init(name.getMethodName(), conf);
328     
329     int storeFileNum = 4;
330     for (int i = 1; i <= storeFileNum; i++) {
331       LOG.info("Adding some data for the store file #"+i);
332       this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
333       this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
334       this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
335       flush(i);
336     }
337     // after flush; check the lowest time stamp
338     long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
339     long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
340     Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
341 
342     // after compact; check the lowest time stamp
343     store.compact(store.requestCompaction());
344     lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
345     lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
346     Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
347   }
348   
349   private static long getLowestTimeStampFromFS(FileSystem fs, 
350       final Collection<StoreFile> candidates) throws IOException {
351     long minTs = Long.MAX_VALUE;
352     if (candidates.isEmpty()) {
353       return minTs; 
354     }
355     Path[] p = new Path[candidates.size()];
356     int i = 0;
357     for (StoreFile sf : candidates) {
358       p[i] = sf.getPath();
359       ++i;
360     }
361     
362     FileStatus[] stats = fs.listStatus(p);
363     if (stats == null || stats.length == 0) {
364       return minTs;
365     }
366     for (FileStatus s : stats) {
367       minTs = Math.min(minTs, s.getModificationTime());
368     }
369     return minTs;
370   }
371 
372   //////////////////////////////////////////////////////////////////////////////
373   // Get tests
374   //////////////////////////////////////////////////////////////////////////////
375 
376   /**
377    * Test for hbase-1686.
378    * @throws IOException
379    */
380   @Test
381   public void testEmptyStoreFile() throws IOException {
382     init(this.name.getMethodName());
383     // Write a store file.
384     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
385     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
386     flush(1);
387     // Now put in place an empty store file.  Its a little tricky.  Have to
388     // do manually with hacked in sequence id.
389     StoreFile f = this.store.getStorefiles().iterator().next();
390     Path storedir = f.getPath().getParent();
391     long seqid = f.getMaxSequenceId();
392     Configuration c = HBaseConfiguration.create();
393     FileSystem fs = FileSystem.get(c);
394     StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
395         fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL)
396             .withOutputDir(storedir)
397             .build();
398     w.appendMetadata(seqid + 1, false);
399     w.close();
400     this.store.close();
401     // Reopen it... should pick up two files
402     this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
403     Assert.assertEquals(2, this.store.getStorefilesCount());
404 
405     result = HBaseTestingUtility.getFromStoreFile(store,
406         get.getRow(),
407         qualifiers);
408     Assert.assertEquals(1, result.size());
409   }
410 
411   /**
412    * Getting data from memstore only
413    * @throws IOException
414    */
415   @Test
416   public void testGet_FromMemStoreOnly() throws IOException {
417     init(this.name.getMethodName());
418 
419     //Put data in memstore
420     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
421     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
422     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
423     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
424     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
425     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
426 
427     //Get
428     result = HBaseTestingUtility.getFromStoreFile(store,
429         get.getRow(), qualifiers);
430 
431     //Compare
432     assertCheck();
433   }
434 
435   /**
436    * Getting data from files only
437    * @throws IOException
438    */
439   @Test
440   public void testGet_FromFilesOnly() throws IOException {
441     init(this.name.getMethodName());
442 
443     //Put data in memstore
444     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
445     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
446     //flush
447     flush(1);
448 
449     //Add more data
450     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
451     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
452     //flush
453     flush(2);
454 
455     //Add more data
456     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
457     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
458     //flush
459     flush(3);
460 
461     //Get
462     result = HBaseTestingUtility.getFromStoreFile(store,
463         get.getRow(),
464         qualifiers);
465     //this.store.get(get, qualifiers, result);
466 
467     //Need to sort the result since multiple files
468     Collections.sort(result, KeyValue.COMPARATOR);
469 
470     //Compare
471     assertCheck();
472   }
473 
474   /**
475    * Getting data from memstore and files
476    * @throws IOException
477    */
478   @Test
479   public void testGet_FromMemStoreAndFiles() throws IOException {
480     init(this.name.getMethodName());
481 
482     //Put data in memstore
483     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
484     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
485     //flush
486     flush(1);
487 
488     //Add more data
489     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
490     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
491     //flush
492     flush(2);
493 
494     //Add more data
495     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
496     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
497 
498     //Get
499     result = HBaseTestingUtility.getFromStoreFile(store,
500         get.getRow(), qualifiers);
501 
502     //Need to sort the result since multiple files
503     Collections.sort(result, KeyValue.COMPARATOR);
504 
505     //Compare
506     assertCheck();
507   }
508 
509   private void flush(int storeFilessize) throws IOException{
510     this.store.snapshot();
511     flushStore(store, id++);
512     Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
513     Assert.assertEquals(0, this.store.memstore.kvset.size());
514   }
515 
516   private void assertCheck() {
517     Assert.assertEquals(expected.size(), result.size());
518     for(int i=0; i<expected.size(); i++) {
519       Assert.assertEquals(expected.get(i), result.get(i));
520     }
521   }
522 
523   //////////////////////////////////////////////////////////////////////////////
524   // IncrementColumnValue tests
525   //////////////////////////////////////////////////////////////////////////////
526   /*
527    * test the internal details of how ICV works, especially during a flush scenario.
528    */
529   @Test
530   public void testIncrementColumnValue_ICVDuringFlush()
531       throws IOException, InterruptedException {
532     init(this.name.getMethodName());
533 
534     long oldValue = 1L;
535     long newValue = 3L;
536     this.store.add(new KeyValue(row, family, qf1,
537         System.currentTimeMillis(),
538         Bytes.toBytes(oldValue)));
539 
540     // snapshot the store.
541     this.store.snapshot();
542 
543     // add other things:
544     this.store.add(new KeyValue(row, family, qf2,
545         System.currentTimeMillis(),
546         Bytes.toBytes(oldValue)));
547 
548     // update during the snapshot.
549     long ret = this.store.updateColumnValue(row, family, qf1, newValue);
550 
551     // memstore should have grown by some amount.
552     Assert.assertTrue(ret > 0);
553 
554     // then flush.
555     flushStore(store, id++);
556     Assert.assertEquals(1, this.store.getStorefiles().size());
557     // from the one we inserted up there, and a new one
558     Assert.assertEquals(2, this.store.memstore.kvset.size());
559 
560     // how many key/values for this row are there?
561     Get get = new Get(row);
562     get.addColumn(family, qf1);
563     get.setMaxVersions(); // all versions.
564     List<Cell> results = new ArrayList<Cell>();
565 
566     results = HBaseTestingUtility.getFromStoreFile(store, get);
567     Assert.assertEquals(2, results.size());
568 
569     long ts1 = results.get(0).getTimestamp();
570     long ts2 = results.get(1).getTimestamp();
571 
572     Assert.assertTrue(ts1 > ts2);
573 
574     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
575     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
576   }
577 
578   @After
579   public void tearDown() throws Exception {
580     EnvironmentEdgeManagerTestHelper.reset();
581   }
582 
583   @Test
584   public void testICV_negMemstoreSize()  throws IOException {
585       init(this.name.getMethodName());
586 
587     long time = 100;
588     ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
589     ee.setValue(time);
590     EnvironmentEdgeManagerTestHelper.injectEdge(ee);
591     long newValue = 3L;
592     long size = 0;
593 
594 
595     size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
596         System.currentTimeMillis(),
597         Bytes.toBytes(newValue)));
598     size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
599         System.currentTimeMillis(),
600         Bytes.toBytes(newValue)));
601     size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
602         System.currentTimeMillis(),
603         Bytes.toBytes(newValue)));
604     size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
605         System.currentTimeMillis(),
606         Bytes.toBytes(newValue)));
607     size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
608         System.currentTimeMillis(),
609         Bytes.toBytes(newValue)));
610 
611 
612     for ( int i = 0 ; i < 10000 ; ++i) {
613       newValue++;
614 
615       long ret = this.store.updateColumnValue(row, family, qf1, newValue);
616       long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
617 
618       if (ret != 0) System.out.println("ret: " + ret);
619       if (ret2 != 0) System.out.println("ret2: " + ret2);
620 
621       Assert.assertTrue("ret: " + ret, ret >= 0);
622       size += ret;
623       Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
624       size += ret2;
625 
626 
627       if (i % 1000 == 0)
628         ee.setValue(++time);
629     }
630 
631     long computedSize=0;
632     for (KeyValue kv : this.store.memstore.kvset) {
633       long kvsize = MemStore.heapSizeChange(kv, true);
634       //System.out.println(kv + " size= " + kvsize + " kvsize= " + kv.heapSize());
635       computedSize += kvsize;
636     }
637     Assert.assertEquals(computedSize, size);
638   }
639 
640   @Test
641   public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
642     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
643     EnvironmentEdgeManagerTestHelper.injectEdge(mee);
644     init(this.name.getMethodName());
645 
646     long oldValue = 1L;
647     long newValue = 3L;
648     this.store.add(new KeyValue(row, family, qf1,
649         EnvironmentEdgeManager.currentTimeMillis(),
650         Bytes.toBytes(oldValue)));
651 
652     // snapshot the store.
653     this.store.snapshot();
654 
655     // update during the snapshot, the exact same TS as the Put (lololol)
656     long ret = this.store.updateColumnValue(row, family, qf1, newValue);
657 
658     // memstore should have grown by some amount.
659     Assert.assertTrue(ret > 0);
660 
661     // then flush.
662     flushStore(store, id++);
663     Assert.assertEquals(1, this.store.getStorefiles().size());
664     Assert.assertEquals(1, this.store.memstore.kvset.size());
665 
666     // now increment again:
667     newValue += 1;
668     this.store.updateColumnValue(row, family, qf1, newValue);
669 
670     // at this point we have a TS=1 in snapshot, and a TS=2 in kvset, so increment again:
671     newValue += 1;
672     this.store.updateColumnValue(row, family, qf1, newValue);
673 
674     // the second TS should be TS=2 or higher., even though 'time=1' right now.
675 
676 
677     // how many key/values for this row are there?
678     Get get = new Get(row);
679     get.addColumn(family, qf1);
680     get.setMaxVersions(); // all versions.
681     List<Cell> results = new ArrayList<Cell>();
682 
683     results = HBaseTestingUtility.getFromStoreFile(store, get);
684     Assert.assertEquals(2, results.size());
685 
686     long ts1 = results.get(0).getTimestamp();
687     long ts2 = results.get(1).getTimestamp();
688 
689     Assert.assertTrue(ts1 > ts2);
690     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
691     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
692 
693     mee.setValue(2); // time goes up slightly
694     newValue += 1;
695     this.store.updateColumnValue(row, family, qf1, newValue);
696 
697     results = HBaseTestingUtility.getFromStoreFile(store, get);
698     Assert.assertEquals(2, results.size());
699 
700     ts1 = results.get(0).getTimestamp();
701     ts2 = results.get(1).getTimestamp();
702 
703     Assert.assertTrue(ts1 > ts2);
704     Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
705     Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
706   }
707 
708   @Test
709   public void testHandleErrorsInFlush() throws Exception {
710     LOG.info("Setting up a faulty file system that cannot write");
711 
712     final Configuration conf = HBaseConfiguration.create();
713     User user = User.createUserForTesting(conf,
714         "testhandleerrorsinflush", new String[]{"foo"});
715     // Inject our faulty LocalFileSystem
716     conf.setClass("fs.file.impl", FaultyFileSystem.class,
717         FileSystem.class);
718     user.runAs(new PrivilegedExceptionAction<Object>() {
719       public Object run() throws Exception {
720         // Make sure it worked (above is sensitive to caching details in hadoop core)
721         FileSystem fs = FileSystem.get(conf);
722         Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
723 
724         // Initialize region
725         init(name.getMethodName(), conf);
726 
727         LOG.info("Adding some data");
728         store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
729         store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
730         store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
731 
732         LOG.info("Before flush, we should have no files");
733 
734         Collection<StoreFileInfo> files =
735           store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
736         Assert.assertEquals(0, files != null ? files.size() : 0);
737 
738         //flush
739         try {
740           LOG.info("Flushing");
741           flush(1);
742           Assert.fail("Didn't bubble up IOE!");
743         } catch (IOException ioe) {
744           Assert.assertTrue(ioe.getMessage().contains("Fault injected"));
745         }
746 
747         LOG.info("After failed flush, we should still have no files!");
748         files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
749         Assert.assertEquals(0, files != null ? files.size() : 0);
750         store.getHRegion().getLog().closeAndDelete();
751         return null;
752       }
753     });
754     FileSystem.closeAllForUGI(user.getUGI());
755   }
756 
757   /**
758    * Faulty file system that will fail if you write past its fault position the FIRST TIME
759    * only; thereafter it will succeed.  Used by {@link TestHRegion} too.
760    */
761   static class FaultyFileSystem extends FilterFileSystem {
762     List<SoftReference<FaultyOutputStream>> outStreams =
763       new ArrayList<SoftReference<FaultyOutputStream>>();
764     private long faultPos = 200;
765     AtomicBoolean fault = new AtomicBoolean(true);
766 
767     public FaultyFileSystem() {
768       super(new LocalFileSystem());
769       System.err.println("Creating faulty!");
770     }
771 
772     @Override
773     public FSDataOutputStream create(Path p) throws IOException {
774       return new FaultyOutputStream(super.create(p), faultPos, fault);
775     }
776 
777     @Override
778     public FSDataOutputStream create(Path f, FsPermission permission,
779         boolean overwrite, int bufferSize, short replication, long blockSize,
780         Progressable progress) throws IOException {
781       return new FaultyOutputStream(super.create(f, permission,
782           overwrite, bufferSize, replication, blockSize, progress), faultPos, fault);
783     }
784 
785     public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
786         int bufferSize, short replication, long blockSize, Progressable progress)
787     throws IOException {
788       // Fake it.  Call create instead.  The default implementation throws an IOE
789       // that this is not supported.
790       return create(f, overwrite, bufferSize, replication, blockSize, progress);
791     }
792   }
793 
794   static class FaultyOutputStream extends FSDataOutputStream {
795     volatile long faultPos = Long.MAX_VALUE;
796     private final AtomicBoolean fault;
797 
798     public FaultyOutputStream(FSDataOutputStream out, long faultPos, final AtomicBoolean fault)
799     throws IOException {
800       super(out, null);
801       this.faultPos = faultPos;
802       this.fault = fault;
803     }
804 
805     @Override
806     public void write(byte[] buf, int offset, int length) throws IOException {
807       System.err.println("faulty stream write at pos " + getPos());
808       injectFault();
809       super.write(buf, offset, length);
810     }
811 
812     private void injectFault() throws IOException {
813       if (this.fault.get() && getPos() >= faultPos) {
814         throw new IOException("Fault injected");
815       }
816     }
817   }
818 
819   private static void flushStore(HStore store, long id) throws IOException {
820     StoreFlushContext storeFlushCtx = store.createFlushContext(id);
821     storeFlushCtx.prepare();
822     storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
823     storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
824   }
825 
826   /**
827    * Generate a list of KeyValues for testing based on given parameters
828    * @param timestamps
829    * @param numRows
830    * @param qualifier
831    * @param family
832    * @return
833    */
834   List<Cell> getKeyValueSet(long[] timestamps, int numRows,
835       byte[] qualifier, byte[] family) {
836     List<Cell> kvList = new ArrayList<Cell>();
837     for (int i=1;i<=numRows;i++) {
838       byte[] b = Bytes.toBytes(i);
839       for (long timestamp: timestamps) {
840         kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
841       }
842     }
843     return kvList;
844   }
845 
846   /**
847    * Test to ensure correctness when using Stores with multiple timestamps
848    * @throws IOException
849    */
850   @Test
851   public void testMultipleTimestamps() throws IOException {
852     int numRows = 1;
853     long[] timestamps1 = new long[] {1,5,10,20};
854     long[] timestamps2 = new long[] {30,80};
855 
856     init(this.name.getMethodName());
857 
858     List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
859     for (Cell kv : kvList1) {
860       this.store.add(KeyValueUtil.ensureKeyValue(kv));
861     }
862 
863     this.store.snapshot();
864     flushStore(store, id++);
865 
866     List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
867     for(Cell kv : kvList2) {
868       this.store.add(KeyValueUtil.ensureKeyValue(kv));
869     }
870 
871     List<Cell> result;
872     Get get = new Get(Bytes.toBytes(1));
873     get.addColumn(family,qf1);
874 
875     get.setTimeRange(0,15);
876     result = HBaseTestingUtility.getFromStoreFile(store, get);
877     Assert.assertTrue(result.size()>0);
878 
879     get.setTimeRange(40,90);
880     result = HBaseTestingUtility.getFromStoreFile(store, get);
881     Assert.assertTrue(result.size()>0);
882 
883     get.setTimeRange(10,45);
884     result = HBaseTestingUtility.getFromStoreFile(store, get);
885     Assert.assertTrue(result.size()>0);
886 
887     get.setTimeRange(80,145);
888     result = HBaseTestingUtility.getFromStoreFile(store, get);
889     Assert.assertTrue(result.size()>0);
890 
891     get.setTimeRange(1,2);
892     result = HBaseTestingUtility.getFromStoreFile(store, get);
893     Assert.assertTrue(result.size()>0);
894 
895     get.setTimeRange(90,200);
896     result = HBaseTestingUtility.getFromStoreFile(store, get);
897     Assert.assertTrue(result.size()==0);
898   }
899 
900   /**
901    * Test for HBASE-3492 - Test split on empty colfam (no store files).
902    *
903    * @throws IOException When the IO operations fail.
904    */
905   @Test
906   public void testSplitWithEmptyColFam() throws IOException {
907     init(this.name.getMethodName());
908     Assert.assertNull(store.getSplitPoint());
909     store.getHRegion().forceSplit(null);
910     Assert.assertNull(store.getSplitPoint());
911     store.getHRegion().clearSplit_TESTS_ONLY();
912   }
913 
914   @Test
915   public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
916     final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
917     long anyValue = 10;
918 
919     // We'll check that it uses correct config and propagates it appropriately by going thru
920     // the simplest "real" path I can find - "throttleCompaction", which just checks whether
921     // a number we pass in is higher than some config value, inside compactionPolicy.
922     Configuration conf = HBaseConfiguration.create();
923     conf.setLong(CONFIG_KEY, anyValue);
924     init(name.getMethodName() + "-xml", conf);
925     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
926     Assert.assertFalse(store.throttleCompaction(anyValue));
927 
928     // HTD overrides XML.
929     --anyValue;
930     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
931     HColumnDescriptor hcd = new HColumnDescriptor(family);
932     htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
933     init(name.getMethodName() + "-htd", conf, htd, hcd);
934     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
935     Assert.assertFalse(store.throttleCompaction(anyValue));
936 
937     // HCD overrides them both.
938     --anyValue;
939     hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
940     init(name.getMethodName() + "-hcd", conf, htd, hcd);
941     Assert.assertTrue(store.throttleCompaction(anyValue + 1));
942     Assert.assertFalse(store.throttleCompaction(anyValue));
943   }
944 
945   public static class DummyStoreEngine extends DefaultStoreEngine {
946     public static DefaultCompactor lastCreatedCompactor = null;
947     @Override
948     protected void createComponents(
949         Configuration conf, Store store, KVComparator comparator) throws IOException {
950       super.createComponents(conf, store, comparator);
951       lastCreatedCompactor = this.compactor;
952     }
953   }
954 
955   @Test
956   public void testStoreUsesSearchEngineOverride() throws Exception {
957     Configuration conf = HBaseConfiguration.create();
958     conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
959     init(this.name.getMethodName(), conf);
960     Assert.assertEquals(DummyStoreEngine.lastCreatedCompactor,
961       this.store.storeEngine.getCompactor());
962   }
963 }