View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.lang.management.ManagementFactory;
23  import java.lang.management.MemoryMXBean;
24  import java.rmi.UnexpectedException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.concurrent.atomic.AtomicReference;
30  
31  import junit.framework.TestCase;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.Cell;
37  import org.apache.hadoop.hbase.CellUtil;
38  import org.apache.hadoop.hbase.HBaseConfiguration;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.KeyValue;
43  import org.apache.hadoop.hbase.KeyValueTestUtil;
44  import org.apache.hadoop.hbase.MediumTests;
45  import org.apache.hadoop.hbase.client.Scan;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.EnvironmentEdge;
48  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
49  import org.junit.experimental.categories.Category;
50  
51  import com.google.common.base.Joiner;
52  import com.google.common.collect.Iterables;
53  import com.google.common.collect.Lists;
54  
55  /** memstore test case */
56  @Category(MediumTests.class)
57  public class TestMemStore extends TestCase {
58    private final Log LOG = LogFactory.getLog(this.getClass());
59    private MemStore memstore;
60    private static final int ROW_COUNT = 10;
61    private static final int QUALIFIER_COUNT = ROW_COUNT;
62    private static final byte [] FAMILY = Bytes.toBytes("column");
63    private static final byte [] CONTENTS = Bytes.toBytes("contents");
64    private static final byte [] BASIC = Bytes.toBytes("basic");
65    private static final String CONTENTSTR = "contentstr";
66    private MultiVersionConsistencyControl mvcc;
67  
68    @Override
69    public void setUp() throws Exception {
70      super.setUp();
71      this.mvcc = new MultiVersionConsistencyControl();
72      this.memstore = new MemStore();
73    }
74  
75    public void testPutSameKey() {
76      byte [] bytes = Bytes.toBytes(getName());
77      KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
78      this.memstore.add(kv);
79      byte [] other = Bytes.toBytes("somethingelse");
80      KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
81      this.memstore.add(samekey);
82      KeyValue found = this.memstore.kvset.first();
83      assertEquals(1, this.memstore.kvset.size());
84      assertTrue(Bytes.toString(found.getValue()), CellUtil.matchingValue(samekey, found));
85    }
86  
87    /**
88     * Test memstore snapshot happening while scanning.
89     * @throws IOException
90     */
91    public void testScanAcrossSnapshot() throws IOException {
92      int rowCount = addRows(this.memstore);
93      List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
94      Scan scan = new Scan();
95      List<Cell> result = new ArrayList<Cell>();
96      MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
97      ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false,
98          0, this.memstore.comparator);
99      ScanType scanType = ScanType.USER_SCAN;
100     StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
101     int count = 0;
102     try {
103       while (s.next(result)) {
104         LOG.info(result);
105         count++;
106         // Row count is same as column count.
107         assertEquals(rowCount, result.size());
108         result.clear();
109       }
110     } finally {
111       s.close();
112     }
113     assertEquals(rowCount, count);
114     for (KeyValueScanner scanner : memstorescanners) {
115       scanner.close();
116     }
117 
118     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
119     memstorescanners = this.memstore.getScanners();
120     // Now assert can count same number even if a snapshot mid-scan.
121     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
122     count = 0;
123     try {
124       while (s.next(result)) {
125         LOG.info(result);
126         // Assert the stuff is coming out in right order.
127         assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
128         count++;
129         // Row count is same as column count.
130         assertEquals(rowCount, result.size());
131         if (count == 2) {
132           this.memstore.snapshot();
133           LOG.info("Snapshotted");
134         }
135         result.clear();
136       }
137     } finally {
138       s.close();
139     }
140     assertEquals(rowCount, count);
141     for (KeyValueScanner scanner : memstorescanners) {
142       scanner.close();
143     }
144     memstorescanners = this.memstore.getScanners();
145     // Assert that new values are seen in kvset as we scan.
146     long ts = System.currentTimeMillis();
147     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
148     count = 0;
149     int snapshotIndex = 5;
150     try {
151       while (s.next(result)) {
152         LOG.info(result);
153         // Assert the stuff is coming out in right order.
154         assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
155         // Row count is same as column count.
156         assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
157         count++;
158         if (count == snapshotIndex) {
159           this.memstore.snapshot();
160           this.memstore.clearSnapshot(this.memstore.getSnapshot());
161           // Added more rows into kvset.  But the scanner wont see these rows.
162           addRows(this.memstore, ts);
163           LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
164         }
165         result.clear();
166       }
167     } finally {
168       s.close();
169     }
170     assertEquals(rowCount, count);
171   }
172 
173   /**
174    * A simple test which verifies the 3 possible states when scanning across snapshot.
175    * @throws IOException
176    * @throws CloneNotSupportedException 
177    */
178   public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
179     // we are going to the scanning across snapshot with two kvs
180     // kv1 should always be returned before kv2
181     final byte[] one = Bytes.toBytes(1);
182     final byte[] two = Bytes.toBytes(2);
183     final byte[] f = Bytes.toBytes("f");
184     final byte[] q = Bytes.toBytes("q");
185     final byte[] v = Bytes.toBytes(3);
186 
187     final KeyValue kv1 = new KeyValue(one, f, q, v);
188     final KeyValue kv2 = new KeyValue(two, f, q, v);
189 
190     // use case 1: both kvs in kvset
191     this.memstore.add(kv1.clone());
192     this.memstore.add(kv2.clone());
193     verifyScanAcrossSnapshot2(kv1, kv2);
194 
195     // use case 2: both kvs in snapshot
196     this.memstore.snapshot();
197     verifyScanAcrossSnapshot2(kv1, kv2);
198 
199     // use case 3: first in snapshot second in kvset
200     this.memstore = new MemStore();
201     this.memstore.add(kv1.clone());
202     this.memstore.snapshot();
203     this.memstore.add(kv2.clone());
204     verifyScanAcrossSnapshot2(kv1, kv2);
205   }
206 
207   private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
208       throws IOException {
209     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
210     List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
211     assertEquals(1, memstorescanners.size());
212     final KeyValueScanner scanner = memstorescanners.get(0);
213     scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
214     assertEquals(kv1, scanner.next());
215     assertEquals(kv2, scanner.next());
216     assertNull(scanner.next());
217   }
218 
219   private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
220       throws IOException {
221     scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
222     List<Cell> returned = Lists.newArrayList();
223 
224     while (true) {
225       KeyValue next = scanner.next();
226       if (next == null) break;
227       returned.add(next);
228     }
229 
230     assertTrue(
231         "Got:\n" + Joiner.on("\n").join(returned) +
232         "\nExpected:\n" + Joiner.on("\n").join(expected),
233         Iterables.elementsEqual(Arrays.asList(expected), returned));
234     assertNull(scanner.peek());
235   }
236 
237   public void testMemstoreConcurrentControl() throws IOException {
238     final byte[] row = Bytes.toBytes(1);
239     final byte[] f = Bytes.toBytes("family");
240     final byte[] q1 = Bytes.toBytes("q1");
241     final byte[] q2 = Bytes.toBytes("q2");
242     final byte[] v = Bytes.toBytes("value");
243 
244     MultiVersionConsistencyControl.WriteEntry w =
245         mvcc.beginMemstoreInsert();
246 
247     KeyValue kv1 = new KeyValue(row, f, q1, v);
248     kv1.setMvccVersion(w.getWriteNumber());
249     memstore.add(kv1);
250 
251     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
252     KeyValueScanner s = this.memstore.getScanners().get(0);
253     assertScannerResults(s, new KeyValue[]{});
254 
255     mvcc.completeMemstoreInsert(w);
256 
257     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
258     s = this.memstore.getScanners().get(0);
259     assertScannerResults(s, new KeyValue[]{kv1});
260 
261     w = mvcc.beginMemstoreInsert();
262     KeyValue kv2 = new KeyValue(row, f, q2, v);
263     kv2.setMvccVersion(w.getWriteNumber());
264     memstore.add(kv2);
265 
266     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
267     s = this.memstore.getScanners().get(0);
268     assertScannerResults(s, new KeyValue[]{kv1});
269 
270     mvcc.completeMemstoreInsert(w);
271 
272     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
273     s = this.memstore.getScanners().get(0);
274     assertScannerResults(s, new KeyValue[]{kv1, kv2});
275   }
276 
277   /**
278    * Regression test for HBASE-2616, HBASE-2670.
279    * When we insert a higher-memstoreTS version of a cell but with
280    * the same timestamp, we still need to provide consistent reads
281    * for the same scanner.
282    */
283   public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
284     final byte[] row = Bytes.toBytes(1);
285     final byte[] f = Bytes.toBytes("family");
286     final byte[] q1 = Bytes.toBytes("q1");
287     final byte[] q2 = Bytes.toBytes("q2");
288     final byte[] v1 = Bytes.toBytes("value1");
289     final byte[] v2 = Bytes.toBytes("value2");
290 
291     // INSERT 1: Write both columns val1
292     MultiVersionConsistencyControl.WriteEntry w =
293         mvcc.beginMemstoreInsert();
294 
295     KeyValue kv11 = new KeyValue(row, f, q1, v1);
296     kv11.setMvccVersion(w.getWriteNumber());
297     memstore.add(kv11);
298 
299     KeyValue kv12 = new KeyValue(row, f, q2, v1);
300     kv12.setMvccVersion(w.getWriteNumber());
301     memstore.add(kv12);
302     mvcc.completeMemstoreInsert(w);
303 
304     // BEFORE STARTING INSERT 2, SEE FIRST KVS
305     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
306     KeyValueScanner s = this.memstore.getScanners().get(0);
307     assertScannerResults(s, new KeyValue[]{kv11, kv12});
308 
309     // START INSERT 2: Write both columns val2
310     w = mvcc.beginMemstoreInsert();
311     KeyValue kv21 = new KeyValue(row, f, q1, v2);
312     kv21.setMvccVersion(w.getWriteNumber());
313     memstore.add(kv21);
314 
315     KeyValue kv22 = new KeyValue(row, f, q2, v2);
316     kv22.setMvccVersion(w.getWriteNumber());
317     memstore.add(kv22);
318 
319     // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
320     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
321     s = this.memstore.getScanners().get(0);
322     assertScannerResults(s, new KeyValue[]{kv11, kv12});
323 
324     // COMPLETE INSERT 2
325     mvcc.completeMemstoreInsert(w);
326 
327     // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
328     // See HBASE-1485 for discussion about what we should do with
329     // the duplicate-TS inserts
330     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
331     s = this.memstore.getScanners().get(0);
332     assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
333   }
334 
335   /**
336    * When we insert a higher-memstoreTS deletion of a cell but with
337    * the same timestamp, we still need to provide consistent reads
338    * for the same scanner.
339    */
340   public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
341     final byte[] row = Bytes.toBytes(1);
342     final byte[] f = Bytes.toBytes("family");
343     final byte[] q1 = Bytes.toBytes("q1");
344     final byte[] q2 = Bytes.toBytes("q2");
345     final byte[] v1 = Bytes.toBytes("value1");
346     // INSERT 1: Write both columns val1
347     MultiVersionConsistencyControl.WriteEntry w =
348         mvcc.beginMemstoreInsert();
349 
350     KeyValue kv11 = new KeyValue(row, f, q1, v1);
351     kv11.setMvccVersion(w.getWriteNumber());
352     memstore.add(kv11);
353 
354     KeyValue kv12 = new KeyValue(row, f, q2, v1);
355     kv12.setMvccVersion(w.getWriteNumber());
356     memstore.add(kv12);
357     mvcc.completeMemstoreInsert(w);
358 
359     // BEFORE STARTING INSERT 2, SEE FIRST KVS
360     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
361     KeyValueScanner s = this.memstore.getScanners().get(0);
362     assertScannerResults(s, new KeyValue[]{kv11, kv12});
363 
364     // START DELETE: Insert delete for one of the columns
365     w = mvcc.beginMemstoreInsert();
366     KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
367         KeyValue.Type.DeleteColumn);
368     kvDel.setMvccVersion(w.getWriteNumber());
369     memstore.add(kvDel);
370 
371     // BEFORE COMPLETING DELETE, SEE FIRST KVS
372     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
373     s = this.memstore.getScanners().get(0);
374     assertScannerResults(s, new KeyValue[]{kv11, kv12});
375 
376     // COMPLETE DELETE
377     mvcc.completeMemstoreInsert(w);
378 
379     // NOW WE SHOULD SEE DELETE
380     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
381     s = this.memstore.getScanners().get(0);
382     assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
383   }
384 
385 
386   private static class ReadOwnWritesTester extends Thread {
387     static final int NUM_TRIES = 1000;
388 
389     final byte[] row;
390 
391     final byte[] f = Bytes.toBytes("family");
392     final byte[] q1 = Bytes.toBytes("q1");
393 
394     final MultiVersionConsistencyControl mvcc;
395     final MemStore memstore;
396 
397     AtomicReference<Throwable> caughtException;
398 
399 
400     public ReadOwnWritesTester(int id,
401                                MemStore memstore,
402                                MultiVersionConsistencyControl mvcc,
403                                AtomicReference<Throwable> caughtException)
404     {
405       this.mvcc = mvcc;
406       this.memstore = memstore;
407       this.caughtException = caughtException;
408       row = Bytes.toBytes(id);
409     }
410 
411     public void run() {
412       try {
413         internalRun();
414       } catch (Throwable t) {
415         caughtException.compareAndSet(null, t);
416       }
417     }
418 
419     private void internalRun() throws IOException {
420       for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
421         MultiVersionConsistencyControl.WriteEntry w =
422           mvcc.beginMemstoreInsert();
423 
424         // Insert the sequence value (i)
425         byte[] v = Bytes.toBytes(i);
426 
427         KeyValue kv = new KeyValue(row, f, q1, i, v);
428         kv.setMvccVersion(w.getWriteNumber());
429         memstore.add(kv);
430         mvcc.completeMemstoreInsert(w);
431 
432         // Assert that we can read back
433         MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
434 
435         KeyValueScanner s = this.memstore.getScanners().get(0);
436         s.seek(kv);
437 
438         KeyValue ret = s.next();
439         assertNotNull("Didnt find own write at all", ret);
440         assertEquals("Didnt read own writes",
441                      kv.getTimestamp(), ret.getTimestamp());
442       }
443     }
444   }
445 
446   public void testReadOwnWritesUnderConcurrency() throws Throwable {
447 
448     int NUM_THREADS = 8;
449 
450     ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
451     AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
452 
453     for (int i = 0; i < NUM_THREADS; i++) {
454       threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
455       threads[i].start();
456     }
457 
458     for (int i = 0; i < NUM_THREADS; i++) {
459       threads[i].join();
460     }
461 
462     if (caught.get() != null) {
463       throw caught.get();
464     }
465   }
466 
467   /**
468    * Test memstore snapshots
469    * @throws IOException
470    */
471   public void testSnapshotting() throws IOException {
472     final int snapshotCount = 5;
473     // Add some rows, run a snapshot. Do it a few times.
474     for (int i = 0; i < snapshotCount; i++) {
475       addRows(this.memstore);
476       runSnapshot(this.memstore);
477       KeyValueSkipListSet ss = this.memstore.getSnapshot();
478       assertEquals("History not being cleared", 0, ss.size());
479     }
480   }
481 
482   public void testMultipleVersionsSimple() throws Exception {
483     MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
484     byte [] row = Bytes.toBytes("testRow");
485     byte [] family = Bytes.toBytes("testFamily");
486     byte [] qf = Bytes.toBytes("testQualifier");
487     long [] stamps = {1,2,3};
488     byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
489         Bytes.toBytes("value2")};
490     KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
491     KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
492     KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
493 
494     m.add(key0);
495     m.add(key1);
496     m.add(key2);
497 
498     assertTrue("Expected memstore to hold 3 values, actually has " +
499         m.kvset.size(), m.kvset.size() == 3);
500   }
501 
502   //////////////////////////////////////////////////////////////////////////////
503   // Get tests
504   //////////////////////////////////////////////////////////////////////////////
505 
506   /** Test getNextRow from memstore
507    * @throws InterruptedException
508    */
509   public void testGetNextRow() throws Exception {
510     MultiVersionConsistencyControl.resetThreadReadPoint();
511     addRows(this.memstore);
512     // Add more versions to make it a little more interesting.
513     Thread.sleep(1);
514     addRows(this.memstore);
515     KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
516     assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
517       new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
518     for (int i = 0; i < ROW_COUNT; i++) {
519       KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
520         System.currentTimeMillis()));
521       if (i + 1 == ROW_COUNT) {
522         assertEquals(nr, null);
523       } else {
524         assertTrue(KeyValue.COMPARATOR.compareRows(nr,
525           new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
526       }
527     }
528     //starting from each row, validate results should contain the starting row
529     for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
530       ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false,
531           0, this.memstore.comparator);
532       ScanType scanType = ScanType.USER_SCAN;
533       InternalScanner scanner = new StoreScanner(new Scan(
534           Bytes.toBytes(startRowId)), scanInfo, scanType, null,
535           memstore.getScanners());
536       List<Cell> results = new ArrayList<Cell>();
537       for (int i = 0; scanner.next(results); i++) {
538         int rowId = startRowId + i;
539         Cell left = results.get(0);
540         byte[] row1 = Bytes.toBytes(rowId);
541         assertTrue("Row name",
542           KeyValue.COMPARATOR.compareRows(left.getRowArray(), left.getRowOffset(), (int) left.getRowLength(), row1, 0, row1.length) == 0);
543         assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
544         List<Cell> row = new ArrayList<Cell>();
545         for (Cell kv : results) {
546           row.add(kv);
547         }
548         isExpectedRowWithoutTimestamps(rowId, row);
549         // Clear out set.  Otherwise row results accumulate.
550         results.clear();
551       }
552     }
553   }
554 
555   public void testGet_memstoreAndSnapShot() throws IOException {
556     byte [] row = Bytes.toBytes("testrow");
557     byte [] fam = Bytes.toBytes("testfamily");
558     byte [] qf1 = Bytes.toBytes("testqualifier1");
559     byte [] qf2 = Bytes.toBytes("testqualifier2");
560     byte [] qf3 = Bytes.toBytes("testqualifier3");
561     byte [] qf4 = Bytes.toBytes("testqualifier4");
562     byte [] qf5 = Bytes.toBytes("testqualifier5");
563     byte [] val = Bytes.toBytes("testval");
564 
565     //Setting up memstore
566     memstore.add(new KeyValue(row, fam ,qf1, val));
567     memstore.add(new KeyValue(row, fam ,qf2, val));
568     memstore.add(new KeyValue(row, fam ,qf3, val));
569     //Creating a snapshot
570     memstore.snapshot();
571     assertEquals(3, memstore.snapshot.size());
572     //Adding value to "new" memstore
573     assertEquals(0, memstore.kvset.size());
574     memstore.add(new KeyValue(row, fam ,qf4, val));
575     memstore.add(new KeyValue(row, fam ,qf5, val));
576     assertEquals(2, memstore.kvset.size());
577   }
578 
579   //////////////////////////////////////////////////////////////////////////////
580   // Delete tests
581   //////////////////////////////////////////////////////////////////////////////
582   public void testGetWithDelete() throws IOException {
583     byte [] row = Bytes.toBytes("testrow");
584     byte [] fam = Bytes.toBytes("testfamily");
585     byte [] qf1 = Bytes.toBytes("testqualifier");
586     byte [] val = Bytes.toBytes("testval");
587 
588     long ts1 = System.nanoTime();
589     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
590     long ts2 = ts1 + 1;
591     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
592     long ts3 = ts2 +1;
593     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
594     memstore.add(put1);
595     memstore.add(put2);
596     memstore.add(put3);
597 
598     assertEquals(3, memstore.kvset.size());
599 
600     KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
601     memstore.delete(del2);
602 
603     List<Cell> expected = new ArrayList<Cell>();
604     expected.add(put3);
605     expected.add(del2);
606     expected.add(put2);
607     expected.add(put1);
608 
609     assertEquals(4, memstore.kvset.size());
610     int i = 0;
611     for(KeyValue kv : memstore.kvset) {
612       assertEquals(expected.get(i++), kv);
613     }
614   }
615 
616   public void testGetWithDeleteColumn() throws IOException {
617     byte [] row = Bytes.toBytes("testrow");
618     byte [] fam = Bytes.toBytes("testfamily");
619     byte [] qf1 = Bytes.toBytes("testqualifier");
620     byte [] val = Bytes.toBytes("testval");
621 
622     long ts1 = System.nanoTime();
623     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
624     long ts2 = ts1 + 1;
625     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
626     long ts3 = ts2 +1;
627     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
628     memstore.add(put1);
629     memstore.add(put2);
630     memstore.add(put3);
631 
632     assertEquals(3, memstore.kvset.size());
633 
634     KeyValue del2 =
635       new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
636     memstore.delete(del2);
637 
638     List<Cell> expected = new ArrayList<Cell>();
639     expected.add(put3);
640     expected.add(del2);
641     expected.add(put2);
642     expected.add(put1);
643 
644 
645     assertEquals(4, memstore.kvset.size());
646     int i = 0;
647     for (KeyValue kv: memstore.kvset) {
648       assertEquals(expected.get(i++), kv);
649     }
650   }
651 
652 
653   public void testGetWithDeleteFamily() throws IOException {
654     byte [] row = Bytes.toBytes("testrow");
655     byte [] fam = Bytes.toBytes("testfamily");
656     byte [] qf1 = Bytes.toBytes("testqualifier1");
657     byte [] qf2 = Bytes.toBytes("testqualifier2");
658     byte [] qf3 = Bytes.toBytes("testqualifier3");
659     byte [] val = Bytes.toBytes("testval");
660     long ts = System.nanoTime();
661 
662     KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
663     KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
664     KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
665     KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
666 
667     memstore.add(put1);
668     memstore.add(put2);
669     memstore.add(put3);
670     memstore.add(put4);
671 
672     KeyValue del =
673       new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
674     memstore.delete(del);
675 
676     List<Cell> expected = new ArrayList<Cell>();
677     expected.add(del);
678     expected.add(put1);
679     expected.add(put2);
680     expected.add(put4);
681     expected.add(put3);
682 
683 
684 
685     assertEquals(5, memstore.kvset.size());
686     int i = 0;
687     for (KeyValue kv: memstore.kvset) {
688       assertEquals(expected.get(i++), kv);
689     }
690   }
691 
692   public void testKeepDeleteInmemstore() {
693     byte [] row = Bytes.toBytes("testrow");
694     byte [] fam = Bytes.toBytes("testfamily");
695     byte [] qf = Bytes.toBytes("testqualifier");
696     byte [] val = Bytes.toBytes("testval");
697     long ts = System.nanoTime();
698     memstore.add(new KeyValue(row, fam, qf, ts, val));
699     KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
700     memstore.delete(delete);
701     assertEquals(2, memstore.kvset.size());
702     assertEquals(delete, memstore.kvset.first());
703   }
704 
705   public void testRetainsDeleteVersion() throws IOException {
706     // add a put to memstore
707     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
708 
709     // now process a specific delete:
710     KeyValue delete = KeyValueTestUtil.create(
711         "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
712     memstore.delete(delete);
713 
714     assertEquals(2, memstore.kvset.size());
715     assertEquals(delete, memstore.kvset.first());
716   }
717   public void testRetainsDeleteColumn() throws IOException {
718     // add a put to memstore
719     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
720 
721     // now process a specific delete:
722     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
723         KeyValue.Type.DeleteColumn, "dont-care");
724     memstore.delete(delete);
725 
726     assertEquals(2, memstore.kvset.size());
727     assertEquals(delete, memstore.kvset.first());
728   }
729   public void testRetainsDeleteFamily() throws IOException {
730     // add a put to memstore
731     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
732 
733     // now process a specific delete:
734     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
735         KeyValue.Type.DeleteFamily, "dont-care");
736     memstore.delete(delete);
737 
738     assertEquals(2, memstore.kvset.size());
739     assertEquals(delete, memstore.kvset.first());
740   }
741 
742   ////////////////////////////////////
743   //Test for timestamps
744   ////////////////////////////////////
745 
746   /**
747    * Test to ensure correctness when using Memstore with multiple timestamps
748    */
749   public void testMultipleTimestamps() throws IOException {
750     long[] timestamps = new long[] {20,10,5,1};
751     Scan scan = new Scan();
752 
753     for (long timestamp: timestamps)
754       addRows(memstore,timestamp);
755 
756     scan.setTimeRange(0, 2);
757     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
758 
759     scan.setTimeRange(20, 82);
760     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
761 
762     scan.setTimeRange(10, 20);
763     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
764 
765     scan.setTimeRange(8, 12);
766     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
767 
768     /*This test is not required for correctness but it should pass when
769      * timestamp range optimization is on*/
770     //scan.setTimeRange(28, 42);
771     //assertTrue(!memstore.shouldSeek(scan));
772   }
773 
774   ////////////////////////////////////
775   //Test for upsert with MSLAB
776   ////////////////////////////////////
777 
778   /**
779    * Test a pathological pattern that shows why we can't currently
780    * use the MSLAB for upsert workloads. This test inserts data
781    * in the following pattern:
782    *
783    * - row0001 through row1000 (fills up one 2M Chunk)
784    * - row0002 through row1001 (fills up another 2M chunk, leaves one reference
785    *   to the first chunk
786    * - row0003 through row1002 (another chunk, another dangling reference)
787    *
788    * This causes OOME pretty quickly if we use MSLAB for upsert
789    * since each 2M chunk is held onto by a single reference.
790    */
791   public void testUpsertMSLAB() throws Exception {
792     Configuration conf = HBaseConfiguration.create();
793     conf.setBoolean(MemStore.USEMSLAB_KEY, true);
794     memstore = new MemStore(conf, KeyValue.COMPARATOR);
795 
796     int ROW_SIZE = 2048;
797     byte[] qualifier = new byte[ROW_SIZE - 4];
798 
799     MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
800     for (int i = 0; i < 3; i++) { System.gc(); }
801     long usageBefore = bean.getHeapMemoryUsage().getUsed();
802 
803     long size = 0;
804     long ts=0;
805 
806     for (int newValue = 0; newValue < 1000; newValue++) {
807       for (int row = newValue; row < newValue + 1000; row++) {
808         byte[] rowBytes = Bytes.toBytes(row);
809         size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
810       }
811     }
812     System.out.println("Wrote " + ts + " vals");
813     for (int i = 0; i < 3; i++) { System.gc(); }
814     long usageAfter = bean.getHeapMemoryUsage().getUsed();
815     System.out.println("Memory used: " + (usageAfter - usageBefore)
816         + " (heapsize: " + memstore.heapSize() +
817         " size: " + size + ")");
818   }
819 
820   //////////////////////////////////////////////////////////////////////////////
821   // Helpers
822   //////////////////////////////////////////////////////////////////////////////
823   private static byte [] makeQualifier(final int i1, final int i2){
824     return Bytes.toBytes(Integer.toString(i1) + ";" +
825         Integer.toString(i2));
826   }
827 
828   /**
829    * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
830    * as older keyvalues are deleted from the memstore.
831    * @throws Exception
832    */
833   public void testUpsertMemstoreSize() throws Exception {
834     Configuration conf = HBaseConfiguration.create();
835     memstore = new MemStore(conf, KeyValue.COMPARATOR);
836     long oldSize = memstore.size.get();
837 
838     List<Cell> l = new ArrayList<Cell>();
839     KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
840     KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
841     KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");
842 
843     kv1.setMvccVersion(1); kv2.setMvccVersion(1);kv3.setMvccVersion(1);
844     l.add(kv1); l.add(kv2); l.add(kv3);
845 
846     this.memstore.upsert(l, 2);// readpoint is 2
847     long newSize = this.memstore.size.get();
848     assert(newSize > oldSize);
849 
850     KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
851     kv4.setMvccVersion(1);
852     l.clear(); l.add(kv4);
853     this.memstore.upsert(l, 3);
854     assertEquals(newSize, this.memstore.size.get());
855     //this.memstore = null;
856   }
857 
858   ////////////////////////////////////
859   // Test for periodic memstore flushes 
860   // based on time of oldest edit
861   ////////////////////////////////////
862 
863   /**
864    * Tests that the timeOfOldestEdit is updated correctly for the 
865    * various edit operations in memstore.
866    * @throws Exception
867    */
868   public void testUpdateToTimeOfOldestEdit() throws Exception {
869     try {
870       EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
871       EnvironmentEdgeManager.injectEdge(edge);
872       MemStore memstore = new MemStore();
873       long t = memstore.timeOfOldestEdit();
874       assertEquals(t, Long.MAX_VALUE);
875 
876       // test the case that the timeOfOldestEdit is updated after a KV add
877       memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
878       t = memstore.timeOfOldestEdit();
879       assertTrue(t == 1234);
880       // snapshot() will reset timeOfOldestEdit. The method will also assert the 
881       // value is reset to Long.MAX_VALUE
882       t = runSnapshot(memstore);
883 
884       // test the case that the timeOfOldestEdit is updated after a KV delete
885       memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
886       t = memstore.timeOfOldestEdit();
887       assertTrue(t == 1234);
888       t = runSnapshot(memstore);
889 
890       // test the case that the timeOfOldestEdit is updated after a KV upsert
891       List<Cell> l = new ArrayList<Cell>();
892       KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
893       kv1.setMvccVersion(100);
894       l.add(kv1);
895       memstore.upsert(l, 1000);
896       t = memstore.timeOfOldestEdit();
897       assertTrue(t == 1234);
898     } finally {
899       EnvironmentEdgeManager.reset();
900     }
901   }
902 
903   /**
904    * Tests the HRegion.shouldFlush method - adds an edit in the memstore
905    * and checks that shouldFlush returns true, and another where it disables
906    * the periodic flush functionality and tests whether shouldFlush returns
907    * false. 
908    * @throws Exception
909    */
910   public void testShouldFlush() throws Exception {
911     Configuration conf = new Configuration();
912     conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
913     checkShouldFlush(conf, true);
914     // test disable flush
915     conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
916     checkShouldFlush(conf, false);
917   }
918 
919   private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
920     try {
921       EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
922       EnvironmentEdgeManager.injectEdge(edge);
923       HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf);
924       HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
925 
926       Map<byte[], Store> stores = region.getStores();
927       assertTrue(stores.size() == 1);
928 
929       Store s = stores.entrySet().iterator().next().getValue();
930       edge.setCurrentTimeMillis(1234);
931       s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
932       edge.setCurrentTimeMillis(1234 + 100);
933       assertTrue(region.shouldFlush() == false);
934       edge.setCurrentTimeMillis(1234 + 10000);
935       assertTrue(region.shouldFlush() == expected);
936     } finally {
937       EnvironmentEdgeManager.reset();
938     }
939   }
940 
941   private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
942     long t = 1234;
943     @Override
944     public long currentTimeMillis() {
945       return t; 
946     }
947     public void setCurrentTimeMillis(long t) {
948       this.t = t;
949     }
950   }
951 
952   /**
953    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
954    * @param hmc Instance to add rows to.
955    * @return How many rows we added.
956    * @throws IOException
957    */
958   private int addRows(final MemStore hmc) {
959     return addRows(hmc, HConstants.LATEST_TIMESTAMP);
960   }
961 
962   /**
963    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
964    * @param hmc Instance to add rows to.
965    * @return How many rows we added.
966    * @throws IOException
967    */
968   private int addRows(final MemStore hmc, final long ts) {
969     for (int i = 0; i < ROW_COUNT; i++) {
970       long timestamp = ts == HConstants.LATEST_TIMESTAMP?
971         System.currentTimeMillis(): ts;
972       for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
973         byte [] row = Bytes.toBytes(i);
974         byte [] qf = makeQualifier(i, ii);
975         hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
976       }
977     }
978     return ROW_COUNT;
979   }
980 
981   private long runSnapshot(final MemStore hmc) throws UnexpectedException {
982     // Save off old state.
983     int oldHistorySize = hmc.getSnapshot().size();
984     hmc.snapshot();
985     KeyValueSkipListSet ss = hmc.getSnapshot();
986     // Make some assertions about what just happened.
987     assertTrue("History size has not increased", oldHistorySize < ss.size());
988     long t = memstore.timeOfOldestEdit();
989     assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
990     hmc.clearSnapshot(ss);
991     return t;
992   }
993 
994   private void isExpectedRowWithoutTimestamps(final int rowIndex,
995       List<Cell> kvs) {
996     int i = 0;
997     for (Cell kv: kvs) {
998       byte[] expectedColname = makeQualifier(rowIndex, i++);
999       assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname));
1000       // Value is column name as bytes.  Usually result is
1001       // 100 bytes in size at least. This is the default size
1002       // for BytesWriteable.  For comparison, convert bytes to
1003       // String and trim to remove trailing null bytes.
1004       assertTrue("Content", CellUtil.matchingValue(kv, expectedColname));
1005     }
1006   }
1007 
1008   private KeyValue getDeleteKV(byte [] row) {
1009     return new KeyValue(row, Bytes.toBytes("test_col"), null,
1010       HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
1011   }
1012 
1013   private KeyValue getKV(byte [] row, byte [] value) {
1014     return new KeyValue(row, Bytes.toBytes("test_col"), null,
1015       HConstants.LATEST_TIMESTAMP, value);
1016   }
1017   private static void addRows(int count, final MemStore mem) {
1018     long nanos = System.nanoTime();
1019 
1020     for (int i = 0 ; i < count ; i++) {
1021       if (i % 1000 == 0) {
1022 
1023         System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
1024         nanos = System.nanoTime();
1025       }
1026       long timestamp = System.currentTimeMillis();
1027 
1028       for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
1029         byte [] row = Bytes.toBytes(i);
1030         byte [] qf = makeQualifier(i, ii);
1031         mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
1032       }
1033     }
1034   }
1035 
1036 
1037   static void doScan(MemStore ms, int iteration) throws IOException {
1038     long nanos = System.nanoTime();
1039     KeyValueScanner s = ms.getScanners().get(0);
1040     s.seek(KeyValue.createFirstOnRow(new byte[]{}));
1041 
1042     System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
1043     int cnt=0;
1044     while(s.next() != null) ++cnt;
1045 
1046     System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
1047 
1048   }
1049 
1050   public static void main(String [] args) throws IOException {
1051     MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
1052     MemStore ms = new MemStore();
1053 
1054     long n1 = System.nanoTime();
1055     addRows(25000, ms);
1056     System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
1057 
1058 
1059     System.out.println("foo");
1060 
1061     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
1062 
1063     for (int i = 0 ; i < 50 ; i++)
1064       doScan(ms, i);
1065 
1066   }
1067 
1068 
1069 
1070 }
1071