View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.lang.ref.SoftReference;
24  import java.security.PrivilegedExceptionAction;
25  import java.util.ArrayList;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.NavigableSet;
31  import java.util.concurrent.ConcurrentSkipListSet;
32  
33  import junit.framework.TestCase;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FSDataOutputStream;
39  import org.apache.hadoop.fs.FileStatus;
40  import org.apache.hadoop.fs.FileSystem;
41  import org.apache.hadoop.fs.FilterFileSystem;
42  import org.apache.hadoop.fs.LocalFileSystem;
43  import org.apache.hadoop.fs.Path;
44  import org.apache.hadoop.fs.permission.FsPermission;
45  import org.apache.hadoop.hbase.Cell;
46  import org.apache.hadoop.hbase.CellUtil;
47  import org.apache.hadoop.hbase.HBaseConfiguration;
48  import org.apache.hadoop.hbase.HBaseTestingUtility;
49  import org.apache.hadoop.hbase.HColumnDescriptor;
50  import org.apache.hadoop.hbase.HRegionInfo;
51  import org.apache.hadoop.hbase.HTableDescriptor;
52  import org.apache.hadoop.hbase.KeyValue;
53  import org.apache.hadoop.hbase.KeyValue.KVComparator;
54  import org.apache.hadoop.hbase.KeyValueUtil;
55  import org.apache.hadoop.hbase.MediumTests;
56  import org.apache.hadoop.hbase.TableName;
57  import org.apache.hadoop.hbase.client.Get;
58  import org.apache.hadoop.hbase.io.compress.Compression;
59  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
60  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
61  import org.apache.hadoop.hbase.io.hfile.HFile;
62  import org.apache.hadoop.hbase.io.hfile.HFileContext;
63  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
64  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
65  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
66  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
67  import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
68  import org.apache.hadoop.hbase.regionserver.wal.HLog;
69  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
70  import org.apache.hadoop.hbase.security.User;
71  import org.apache.hadoop.hbase.util.Bytes;
72  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
74  import org.apache.hadoop.hbase.util.FSUtils;
75  import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
76  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
77  import org.apache.hadoop.util.Progressable;
78  import org.junit.experimental.categories.Category;
79  import org.mockito.Mockito;
80  
81  /**
82   * Test class for the Store
83   */
84  @Category(MediumTests.class)
85  public class TestStore extends TestCase {
86    public static final Log LOG = LogFactory.getLog(TestStore.class);
87  
88    HStore store;
89    byte [] table = Bytes.toBytes("table");
90    byte [] family = Bytes.toBytes("family");
91  
92    byte [] row = Bytes.toBytes("row");
93    byte [] row2 = Bytes.toBytes("row2");
94    byte [] qf1 = Bytes.toBytes("qf1");
95    byte [] qf2 = Bytes.toBytes("qf2");
96    byte [] qf3 = Bytes.toBytes("qf3");
97    byte [] qf4 = Bytes.toBytes("qf4");
98    byte [] qf5 = Bytes.toBytes("qf5");
99    byte [] qf6 = Bytes.toBytes("qf6");
100 
101   NavigableSet<byte[]> qualifiers =
102     new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
103 
104   List<Cell> expected = new ArrayList<Cell>();
105   List<Cell> result = new ArrayList<Cell>();
106 
107   long id = System.currentTimeMillis();
108   Get get = new Get(row);
109 
110   private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
111   private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString();
112 
113 
114   /**
115    * Setup
116    * @throws IOException
117    */
118   @Override
119   public void setUp() throws IOException {
120     qualifiers.add(qf1);
121     qualifiers.add(qf3);
122     qualifiers.add(qf5);
123 
124     Iterator<byte[]> iter = qualifiers.iterator();
125     while(iter.hasNext()){
126       byte [] next = iter.next();
127       expected.add(new KeyValue(row, family, next, 1, (byte[])null));
128       get.addColumn(family, next);
129     }
130   }
131 
132   private void init(String methodName) throws IOException {
133     init(methodName, HBaseConfiguration.create());
134   }
135 
136   private void init(String methodName, Configuration conf)
137   throws IOException {
138     HColumnDescriptor hcd = new HColumnDescriptor(family);
139     // some of the tests write 4 versions and then flush
140     // (with HBASE-4241, lower versions are collected on flush)
141     hcd.setMaxVersions(4);
142     init(methodName, conf, hcd);
143   }
144 
145   private void init(String methodName, Configuration conf,
146       HColumnDescriptor hcd) throws IOException {
147     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
148     init(methodName, conf, htd, hcd);
149   }
150 
151   @SuppressWarnings("deprecation")
152   private void init(String methodName, Configuration conf, HTableDescriptor htd,
153       HColumnDescriptor hcd) throws IOException {
154     //Setting up a Store
155     Path basedir = new Path(DIR+methodName);
156     Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
157     String logName = "logs";
158     Path logdir = new Path(basedir, logName);
159 
160     FileSystem fs = FileSystem.get(conf);
161 
162     fs.delete(logdir, true);
163 
164     htd.addFamily(hcd);
165     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
166     HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
167     HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
168 
169     store = new HStore(region, hcd, conf);
170   }
171 
172   /**
173    * Verify that compression and data block encoding are respected by the
174    * Store.createWriterInTmp() method, used on store flush.
175    */
176   public void testCreateWriter() throws Exception {
177     Configuration conf = HBaseConfiguration.create();
178     FileSystem fs = FileSystem.get(conf);
179 
180     HColumnDescriptor hcd = new HColumnDescriptor(family);
181     hcd.setCompressionType(Compression.Algorithm.GZ);
182     hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
183     init(getName(), conf, hcd);
184 
185     // Test createWriterInTmp()
186     StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
187     Path path = writer.getPath();
188     writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
189     writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
190     writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
191     writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
192     writer.close();
193 
194     // Verify that compression and encoding settings are respected
195     HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
196     assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
197     assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
198     reader.close();
199   }
200 
201   public void testDeleteExpiredStoreFiles() throws Exception {
202     int storeFileNum = 4;
203     int ttl = 4;
204     IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
205     EnvironmentEdgeManagerTestHelper.injectEdge(edge);
206     
207     Configuration conf = HBaseConfiguration.create();
208     // Enable the expired store file deletion
209     conf.setBoolean("hbase.store.delete.expired.storefile", true);
210     HColumnDescriptor hcd = new HColumnDescriptor(family);
211     hcd.setTimeToLive(ttl);
212     init(getName(), conf, hcd);
213 
214     long sleepTime = this.store.getScanInfo().getTtl() / storeFileNum;
215     long timeStamp;
216     // There are 4 store files and the max time stamp difference among these
217     // store files will be (this.store.ttl / storeFileNum)
218     for (int i = 1; i <= storeFileNum; i++) {
219       LOG.info("Adding some data for the store file #" + i);
220       timeStamp = EnvironmentEdgeManager.currentTimeMillis();
221       this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null));
222       this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null));
223       this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null));
224       flush(i);
225       edge.incrementTime(sleepTime);
226     }
227 
228     // Verify the total number of store files
229     assertEquals(storeFileNum, this.store.getStorefiles().size());
230 
231     // Each compaction request will find one expired store file and delete it
232     // by the compaction.
233     for (int i = 1; i <= storeFileNum; i++) {
234       // verify the expired store file.
235       CompactionContext compaction = this.store.requestCompaction();
236       CompactionRequest cr = compaction.getRequest();
237       // the first is expired normally.
238       // If not the first compaction, there is another empty store file,
239       List<StoreFile> files = new ArrayList<StoreFile>(cr.getFiles());
240       assertEquals(Math.min(i, 2), cr.getFiles().size());
241       for (int j = 0; j < files.size(); j++) {
242         assertTrue(files.get(j).getReader().getMaxTimestamp() < (edge
243             .currentTimeMillis() - this.store.getScanInfo().getTtl()));
244       }
245       // Verify that the expired store file is compacted to an empty store file.
246       // Default compaction policy creates just one and only one compacted file.
247       StoreFile compactedFile = this.store.compact(compaction).get(0);
248       // It is an empty store file.
249       assertEquals(0, compactedFile.getReader().getEntries());
250 
251       // Let the next store file expired.
252       edge.incrementTime(sleepTime);
253     }
254   }
255 
256   public void testLowestModificationTime() throws Exception {
257     Configuration conf = HBaseConfiguration.create();
258     FileSystem fs = FileSystem.get(conf);
259     // Initialize region
260     init(getName(), conf);
261     
262     int storeFileNum = 4;
263     for (int i = 1; i <= storeFileNum; i++) {
264       LOG.info("Adding some data for the store file #"+i);
265       this.store.add(new KeyValue(row, family, qf1, i, (byte[])null));
266       this.store.add(new KeyValue(row, family, qf2, i, (byte[])null));
267       this.store.add(new KeyValue(row, family, qf3, i, (byte[])null));
268       flush(i);
269     }
270     // after flush; check the lowest time stamp
271     long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
272     long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
273     assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS);
274 
275     // after compact; check the lowest time stamp
276     store.compact(store.requestCompaction());
277     lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles());
278     lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles());
279     assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS);
280   }
281   
282   private static long getLowestTimeStampFromFS(FileSystem fs, 
283       final Collection<StoreFile> candidates) throws IOException {
284     long minTs = Long.MAX_VALUE;
285     if (candidates.isEmpty()) {
286       return minTs; 
287     }
288     Path[] p = new Path[candidates.size()];
289     int i = 0;
290     for (StoreFile sf : candidates) {
291       p[i] = sf.getPath();
292       ++i;
293     }
294     
295     FileStatus[] stats = fs.listStatus(p);
296     if (stats == null || stats.length == 0) {
297       return minTs;
298     }
299     for (FileStatus s : stats) {
300       minTs = Math.min(minTs, s.getModificationTime());
301     }
302     return minTs;
303   }
304 
305   //////////////////////////////////////////////////////////////////////////////
306   // Get tests
307   //////////////////////////////////////////////////////////////////////////////
308 
309   private static final int BLOCKSIZE_SMALL = 8192;
310   /**
311    * Test for hbase-1686.
312    * @throws IOException
313    */
314   public void testEmptyStoreFile() throws IOException {
315     init(this.getName());
316     // Write a store file.
317     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
318     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
319     flush(1);
320     // Now put in place an empty store file.  Its a little tricky.  Have to
321     // do manually with hacked in sequence id.
322     StoreFile f = this.store.getStorefiles().iterator().next();
323     Path storedir = f.getPath().getParent();
324     long seqid = f.getMaxSequenceId();
325     Configuration c = HBaseConfiguration.create();
326     FileSystem fs = FileSystem.get(c);
327     HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
328     StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
329         fs)
330             .withOutputDir(storedir)
331             .withFileContext(meta)
332             .build();
333     w.appendMetadata(seqid + 1, false);
334     w.close();
335     this.store.close();
336     // Reopen it... should pick up two files
337     this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
338     assertEquals(2, this.store.getStorefilesCount());
339 
340     result = HBaseTestingUtility.getFromStoreFile(store,
341         get.getRow(),
342         qualifiers);
343     assertEquals(1, result.size());
344   }
345 
346   /**
347    * Getting data from memstore only
348    * @throws IOException
349    */
350   public void testGet_FromMemStoreOnly() throws IOException {
351     init(this.getName());
352 
353     //Put data in memstore
354     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
355     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
356     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
357     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
358     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
359     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
360 
361     //Get
362     result = HBaseTestingUtility.getFromStoreFile(store,
363         get.getRow(), qualifiers);
364 
365     //Compare
366     assertCheck();
367   }
368 
369   /**
370    * Getting data from files only
371    * @throws IOException
372    */
373   public void testGet_FromFilesOnly() throws IOException {
374     init(this.getName());
375 
376     //Put data in memstore
377     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
378     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
379     //flush
380     flush(1);
381 
382     //Add more data
383     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
384     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
385     //flush
386     flush(2);
387 
388     //Add more data
389     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
390     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
391     //flush
392     flush(3);
393 
394     //Get
395     result = HBaseTestingUtility.getFromStoreFile(store,
396         get.getRow(),
397         qualifiers);
398     //this.store.get(get, qualifiers, result);
399 
400     //Need to sort the result since multiple files
401     Collections.sort(result, KeyValue.COMPARATOR);
402 
403     //Compare
404     assertCheck();
405   }
406 
407   /**
408    * Getting data from memstore and files
409    * @throws IOException
410    */
411   public void testGet_FromMemStoreAndFiles() throws IOException {
412     init(this.getName());
413 
414     //Put data in memstore
415     this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
416     this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
417     //flush
418     flush(1);
419 
420     //Add more data
421     this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
422     this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
423     //flush
424     flush(2);
425 
426     //Add more data
427     this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
428     this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
429 
430     //Get
431     result = HBaseTestingUtility.getFromStoreFile(store,
432         get.getRow(), qualifiers);
433 
434     //Need to sort the result since multiple files
435     Collections.sort(result, KeyValue.COMPARATOR);
436 
437     //Compare
438     assertCheck();
439   }
440 
441   private void flush(int storeFilessize) throws IOException{
442     this.store.snapshot();
443     flushStore(store, id++);
444     assertEquals(storeFilessize, this.store.getStorefiles().size());
445     assertEquals(0, this.store.memstore.kvset.size());
446   }
447 
448   private void assertCheck() {
449     assertEquals(expected.size(), result.size());
450     for(int i=0; i<expected.size(); i++) {
451       assertEquals(expected.get(i), result.get(i));
452     }
453   }
454 
455   //////////////////////////////////////////////////////////////////////////////
456   // IncrementColumnValue tests
457   //////////////////////////////////////////////////////////////////////////////
458   /*
459    * test the internal details of how ICV works, especially during a flush scenario.
460    */
461   public void testIncrementColumnValue_ICVDuringFlush()
462       throws IOException, InterruptedException {
463     init(this.getName());
464 
465     long oldValue = 1L;
466     long newValue = 3L;
467     this.store.add(new KeyValue(row, family, qf1,
468         System.currentTimeMillis(),
469         Bytes.toBytes(oldValue)));
470 
471     // snapshot the store.
472     this.store.snapshot();
473 
474     // add other things:
475     this.store.add(new KeyValue(row, family, qf2,
476         System.currentTimeMillis(),
477         Bytes.toBytes(oldValue)));
478 
479     // update during the snapshot.
480     long ret = this.store.updateColumnValue(row, family, qf1, newValue);
481 
482     // memstore should have grown by some amount.
483     assertTrue(ret > 0);
484 
485     // then flush.
486     flushStore(store, id++);
487     assertEquals(1, this.store.getStorefiles().size());
488     // from the one we inserted up there, and a new one
489     assertEquals(2, this.store.memstore.kvset.size());
490 
491     // how many key/values for this row are there?
492     Get get = new Get(row);
493     get.addColumn(family, qf1);
494     get.setMaxVersions(); // all versions.
495     List<Cell> results = new ArrayList<Cell>();
496 
497     results = HBaseTestingUtility.getFromStoreFile(store, get);
498     assertEquals(2, results.size());
499 
500     long ts1 = results.get(0).getTimestamp();
501     long ts2 = results.get(1).getTimestamp();
502 
503     assertTrue(ts1 > ts2);
504 
505     assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
506     assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
507   }
508 
509   @Override
510   protected void tearDown() throws Exception {
511     super.tearDown();
512     EnvironmentEdgeManagerTestHelper.reset();
513   }
514 
515   public void testICV_negMemstoreSize()  throws IOException {
516       init(this.getName());
517 
518     long time = 100;
519     ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
520     ee.setValue(time);
521     EnvironmentEdgeManagerTestHelper.injectEdge(ee);
522     long newValue = 3L;
523     long size = 0;
524 
525 
526     size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
527         System.currentTimeMillis(),
528         Bytes.toBytes(newValue)));
529     size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
530         System.currentTimeMillis(),
531         Bytes.toBytes(newValue)));
532     size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
533         System.currentTimeMillis(),
534         Bytes.toBytes(newValue)));
535     size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
536         System.currentTimeMillis(),
537         Bytes.toBytes(newValue)));
538     size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
539         System.currentTimeMillis(),
540         Bytes.toBytes(newValue)));
541 
542 
543     for ( int i = 0 ; i < 10000 ; ++i) {
544       newValue++;
545 
546       long ret = this.store.updateColumnValue(row, family, qf1, newValue);
547       long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);
548 
549       if (ret != 0) System.out.println("ret: " + ret);
550       if (ret2 != 0) System.out.println("ret2: " + ret2);
551 
552       assertTrue("ret: " + ret, ret >= 0);
553       size += ret;
554       assertTrue("ret2: " + ret2, ret2 >= 0);
555       size += ret2;
556 
557 
558       if (i % 1000 == 0)
559         ee.setValue(++time);
560     }
561 
562     long computedSize=0;
563     for (KeyValue kv : this.store.memstore.kvset) {
564       long kvsize = MemStore.heapSizeChange(kv, true);
565       //System.out.println(kv + " size= " + kvsize + " kvsize= " + kv.heapSize());
566       computedSize += kvsize;
567     }
568     assertEquals(computedSize, size);
569   }
570 
571   public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
572     ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
573     EnvironmentEdgeManagerTestHelper.injectEdge(mee);
574     init(this.getName());
575 
576     long oldValue = 1L;
577     long newValue = 3L;
578     this.store.add(new KeyValue(row, family, qf1,
579         EnvironmentEdgeManager.currentTimeMillis(),
580         Bytes.toBytes(oldValue)));
581 
582     // snapshot the store.
583     this.store.snapshot();
584 
585     // update during the snapshot, the exact same TS as the Put (lololol)
586     long ret = this.store.updateColumnValue(row, family, qf1, newValue);
587 
588     // memstore should have grown by some amount.
589     assertTrue(ret > 0);
590 
591     // then flush.
592     flushStore(store, id++);
593     assertEquals(1, this.store.getStorefiles().size());
594     assertEquals(1, this.store.memstore.kvset.size());
595 
596     // now increment again:
597     newValue += 1;
598     this.store.updateColumnValue(row, family, qf1, newValue);
599 
600     // at this point we have a TS=1 in snapshot, and a TS=2 in kvset, so increment again:
601     newValue += 1;
602     this.store.updateColumnValue(row, family, qf1, newValue);
603 
604     // the second TS should be TS=2 or higher., even though 'time=1' right now.
605 
606 
607     // how many key/values for this row are there?
608     Get get = new Get(row);
609     get.addColumn(family, qf1);
610     get.setMaxVersions(); // all versions.
611     List<Cell> results = new ArrayList<Cell>();
612 
613     results = HBaseTestingUtility.getFromStoreFile(store, get);
614     assertEquals(2, results.size());
615 
616     long ts1 = results.get(0).getTimestamp();
617     long ts2 = results.get(1).getTimestamp();
618 
619     assertTrue(ts1 > ts2);
620     assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
621     assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
622 
623     mee.setValue(2); // time goes up slightly
624     newValue += 1;
625     this.store.updateColumnValue(row, family, qf1, newValue);
626 
627     results = HBaseTestingUtility.getFromStoreFile(store, get);
628     assertEquals(2, results.size());
629 
630     ts1 = results.get(0).getTimestamp();
631     ts2 = results.get(1).getTimestamp();
632 
633     assertTrue(ts1 > ts2);
634     assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
635     assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));
636   }
637 
638   public void testHandleErrorsInFlush() throws Exception {
639     LOG.info("Setting up a faulty file system that cannot write");
640 
641     final Configuration conf = HBaseConfiguration.create();
642     User user = User.createUserForTesting(conf,
643         "testhandleerrorsinflush", new String[]{"foo"});
644     // Inject our faulty LocalFileSystem
645     conf.setClass("fs.file.impl", FaultyFileSystem.class,
646         FileSystem.class);
647     user.runAs(new PrivilegedExceptionAction<Object>() {
648       public Object run() throws Exception {
649         // Make sure it worked (above is sensitive to caching details in hadoop core)
650         FileSystem fs = FileSystem.get(conf);
651         assertEquals(FaultyFileSystem.class, fs.getClass());
652 
653         // Initialize region
654         init(getName(), conf);
655 
656         LOG.info("Adding some data");
657         store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
658         store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
659         store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
660 
661         LOG.info("Before flush, we should have no files");
662 
663         Collection<StoreFileInfo> files =
664           store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
665         assertEquals(0, files != null ? files.size() : 0);
666 
667         //flush
668         try {
669           LOG.info("Flushing");
670           flush(1);
671           fail("Didn't bubble up IOE!");
672         } catch (IOException ioe) {
673           assertTrue(ioe.getMessage().contains("Fault injected"));
674         }
675 
676         LOG.info("After failed flush, we should still have no files!");
677         files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName());
678         assertEquals(0, files != null ? files.size() : 0);
679         return null;
680       }
681     });
682   }
683 
684 
685   static class FaultyFileSystem extends FilterFileSystem {
686     List<SoftReference<FaultyOutputStream>> outStreams =
687       new ArrayList<SoftReference<FaultyOutputStream>>();
688     private long faultPos = 200;
689 
690     public FaultyFileSystem() {
691       super(new LocalFileSystem());
692       System.err.println("Creating faulty!");
693     }
694 
695     @Override
696     public FSDataOutputStream create(Path p) throws IOException {
697       return new FaultyOutputStream(super.create(p), faultPos);
698     }
699 
700     @Override
701     public FSDataOutputStream create(Path f, FsPermission permission,
702         boolean overwrite, int bufferSize, short replication, long blockSize,
703         Progressable progress) throws IOException {
704       return new FaultyOutputStream(super.create(f, permission,
705           overwrite, bufferSize, replication, blockSize, progress), faultPos);
706     }
707 
708     public FSDataOutputStream createNonRecursive(Path f, boolean overwrite,
709         int bufferSize, short replication, long blockSize, Progressable progress)
710     throws IOException {
711       // Fake it.  Call create instead.  The default implementation throws an IOE
712       // that this is not supported.
713       return create(f, overwrite, bufferSize, replication, blockSize, progress);
714     }
715   }
716 
717   static class FaultyOutputStream extends FSDataOutputStream {
718     volatile long faultPos = Long.MAX_VALUE;
719 
720     public FaultyOutputStream(FSDataOutputStream out,
721         long faultPos) throws IOException {
722       super(out, null);
723       this.faultPos = faultPos;
724     }
725 
726     @Override
727     public void write(byte[] buf, int offset, int length) throws IOException {
728       System.err.println("faulty stream write at pos " + getPos());
729       injectFault();
730       super.write(buf, offset, length);
731     }
732 
733     private void injectFault() throws IOException {
734       if (getPos() >= faultPos) {
735         throw new IOException("Fault injected");
736       }
737     }
738   }
739 
740 
741 
742   private static void flushStore(HStore store, long id) throws IOException {
743     StoreFlushContext storeFlushCtx = store.createFlushContext(id);
744     storeFlushCtx.prepare();
745     storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
746     storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
747   }
748 
749 
750 
751   /**
752    * Generate a list of KeyValues for testing based on given parameters
753    * @param timestamps
754    * @param numRows
755    * @param qualifier
756    * @param family
757    * @return
758    */
759   List<Cell> getKeyValueSet(long[] timestamps, int numRows,
760       byte[] qualifier, byte[] family) {
761     List<Cell> kvList = new ArrayList<Cell>();
762     for (int i=1;i<=numRows;i++) {
763       byte[] b = Bytes.toBytes(i);
764       for (long timestamp: timestamps) {
765         kvList.add(new KeyValue(b, family, qualifier, timestamp, b));
766       }
767     }
768     return kvList;
769   }
770 
771   /**
772    * Test to ensure correctness when using Stores with multiple timestamps
773    * @throws IOException
774    */
775   public void testMultipleTimestamps() throws IOException {
776     int numRows = 1;
777     long[] timestamps1 = new long[] {1,5,10,20};
778     long[] timestamps2 = new long[] {30,80};
779 
780     init(this.getName());
781 
782     List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family);
783     for (Cell kv : kvList1) {
784       this.store.add(KeyValueUtil.ensureKeyValue(kv));
785     }
786 
787     this.store.snapshot();
788     flushStore(store, id++);
789 
790     List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family);
791     for(Cell kv : kvList2) {
792       this.store.add(KeyValueUtil.ensureKeyValue(kv));
793     }
794 
795     List<Cell> result;
796     Get get = new Get(Bytes.toBytes(1));
797     get.addColumn(family,qf1);
798 
799     get.setTimeRange(0,15);
800     result = HBaseTestingUtility.getFromStoreFile(store, get);
801     assertTrue(result.size()>0);
802 
803     get.setTimeRange(40,90);
804     result = HBaseTestingUtility.getFromStoreFile(store, get);
805     assertTrue(result.size()>0);
806 
807     get.setTimeRange(10,45);
808     result = HBaseTestingUtility.getFromStoreFile(store, get);
809     assertTrue(result.size()>0);
810 
811     get.setTimeRange(80,145);
812     result = HBaseTestingUtility.getFromStoreFile(store, get);
813     assertTrue(result.size()>0);
814 
815     get.setTimeRange(1,2);
816     result = HBaseTestingUtility.getFromStoreFile(store, get);
817     assertTrue(result.size()>0);
818 
819     get.setTimeRange(90,200);
820     result = HBaseTestingUtility.getFromStoreFile(store, get);
821     assertTrue(result.size()==0);
822   }
823 
824   /**
825    * Test for HBASE-3492 - Test split on empty colfam (no store files).
826    *
827    * @throws IOException When the IO operations fail.
828    */
829   public void testSplitWithEmptyColFam() throws IOException {
830     init(this.getName());
831     assertNull(store.getSplitPoint());
832     store.getHRegion().forceSplit(null);
833     assertNull(store.getSplitPoint());
834     store.getHRegion().clearSplit_TESTS_ONLY();
835   }
836 
837   public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception {
838     final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle";
839     long anyValue = 10;
840 
841     // We'll check that it uses correct config and propagates it appropriately by going thru
842     // the simplest "real" path I can find - "throttleCompaction", which just checks whether
843     // a number we pass in is higher than some config value, inside compactionPolicy.
844     Configuration conf = HBaseConfiguration.create();
845     conf.setLong(CONFIG_KEY, anyValue);
846     init(getName() + "-xml", conf);
847     assertTrue(store.throttleCompaction(anyValue + 1));
848     assertFalse(store.throttleCompaction(anyValue));
849 
850     // HTD overrides XML.
851     --anyValue;
852     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
853     HColumnDescriptor hcd = new HColumnDescriptor(family);
854     htd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
855     init(getName() + "-htd", conf, htd, hcd);
856     assertTrue(store.throttleCompaction(anyValue + 1));
857     assertFalse(store.throttleCompaction(anyValue));
858 
859     // HCD overrides them both.
860     --anyValue;
861     hcd.setConfiguration(CONFIG_KEY, Long.toString(anyValue));
862     init(getName() + "-hcd", conf, htd, hcd);
863     assertTrue(store.throttleCompaction(anyValue + 1));
864     assertFalse(store.throttleCompaction(anyValue));
865   }
866 
867   public static class DummyStoreEngine extends DefaultStoreEngine {
868     public static DefaultCompactor lastCreatedCompactor = null;
869     @Override
870     protected void createComponents(
871         Configuration conf, Store store, KVComparator comparator) throws IOException {
872       super.createComponents(conf, store, comparator);
873       lastCreatedCompactor = this.compactor;
874     }
875   }
876 
877   public void testStoreUsesSearchEngineOverride() throws Exception {
878     Configuration conf = HBaseConfiguration.create();
879     conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
880     init(this.getName(), conf);
881     assertEquals(DummyStoreEngine.lastCreatedCompactor, this.store.storeEngine.getCompactor());
882   }
883 }
884