View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.lang.management.ManagementFactory;
23  import java.lang.management.MemoryMXBean;
24  import java.rmi.UnexpectedException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.concurrent.atomic.AtomicReference;
30  
31  import junit.framework.TestCase;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.*;
37  import org.apache.hadoop.hbase.client.Scan;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.EnvironmentEdge;
40  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
41  
42  import com.google.common.base.Joiner;
43  import com.google.common.collect.Iterables;
44  import com.google.common.collect.Lists;
45  import org.junit.experimental.categories.Category;
46  
47  /** memstore test case */
48  @Category(MediumTests.class)
49  public class TestMemStore extends TestCase {
50    private final Log LOG = LogFactory.getLog(this.getClass());
51    private MemStore memstore;
52    private static final int ROW_COUNT = 10;
53    private static final int QUALIFIER_COUNT = ROW_COUNT;
54    private static final byte [] FAMILY = Bytes.toBytes("column");
55    private static final byte [] CONTENTS = Bytes.toBytes("contents");
56    private static final byte [] BASIC = Bytes.toBytes("basic");
57    private static final String CONTENTSTR = "contentstr";
58    private MultiVersionConsistencyControl mvcc;
59  
60    @Override
61    public void setUp() throws Exception {
62      super.setUp();
63      this.mvcc = new MultiVersionConsistencyControl();
64      this.memstore = new MemStore();
65    }
66  
67    public void testPutSameKey() {
68      byte [] bytes = Bytes.toBytes(getName());
69      KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
70      this.memstore.add(kv);
71      byte [] other = Bytes.toBytes("somethingelse");
72      KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
73      this.memstore.add(samekey);
74      KeyValue found = this.memstore.kvset.first();
75      assertEquals(1, this.memstore.kvset.size());
76      assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(),
77        found.getValue()));
78    }
79  
80    /**
81     * Test memstore snapshot happening while scanning.
82     * @throws IOException
83     */
84    public void testScanAcrossSnapshot() throws IOException {
85      int rowCount = addRows(this.memstore);
86      List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
87      Scan scan = new Scan();
88      List<KeyValue> result = new ArrayList<KeyValue>();
89      MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
90      ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false,
91          0, this.memstore.comparator);
92      ScanType scanType = ScanType.USER_SCAN;
93      StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
94      int count = 0;
95      try {
96        while (s.next(result)) {
97          LOG.info(result);
98          count++;
99          // Row count is same as column count.
100         assertEquals(rowCount, result.size());
101         result.clear();
102       }
103     } finally {
104       s.close();
105     }
106     assertEquals(rowCount, count);
107     for (KeyValueScanner scanner : memstorescanners) {
108       scanner.close();
109     }
110 
111     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
112     memstorescanners = this.memstore.getScanners();
113     // Now assert can count same number even if a snapshot mid-scan.
114     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
115     count = 0;
116     try {
117       while (s.next(result)) {
118         LOG.info(result);
119         // Assert the stuff is coming out in right order.
120         assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
121         count++;
122         // Row count is same as column count.
123         assertEquals(rowCount, result.size());
124         if (count == 2) {
125           this.memstore.snapshot();
126           LOG.info("Snapshotted");
127         }
128         result.clear();
129       }
130     } finally {
131       s.close();
132     }
133     assertEquals(rowCount, count);
134     for (KeyValueScanner scanner : memstorescanners) {
135       scanner.close();
136     }
137     memstorescanners = this.memstore.getScanners();
138     // Assert that new values are seen in kvset as we scan.
139     long ts = System.currentTimeMillis();
140     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
141     count = 0;
142     int snapshotIndex = 5;
143     try {
144       while (s.next(result)) {
145         LOG.info(result);
146         // Assert the stuff is coming out in right order.
147         assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
148         // Row count is same as column count.
149         assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
150         count++;
151         if (count == snapshotIndex) {
152           this.memstore.snapshot();
153           this.memstore.clearSnapshot(this.memstore.getSnapshot());
154           // Added more rows into kvset.  But the scanner wont see these rows.
155           addRows(this.memstore, ts);
156           LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
157         }
158         result.clear();
159       }
160     } finally {
161       s.close();
162     }
163     assertEquals(rowCount, count);
164   }
165 
166   /**
167    * A simple test which verifies the 3 possible states when scanning across snapshot.
168    * @throws IOException
169    * @throws CloneNotSupportedException 
170    */
171   public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
172     // we are going to the scanning across snapshot with two kvs
173     // kv1 should always be returned before kv2
174     final byte[] one = Bytes.toBytes(1);
175     final byte[] two = Bytes.toBytes(2);
176     final byte[] f = Bytes.toBytes("f");
177     final byte[] q = Bytes.toBytes("q");
178     final byte[] v = Bytes.toBytes(3);
179 
180     final KeyValue kv1 = new KeyValue(one, f, q, v);
181     final KeyValue kv2 = new KeyValue(two, f, q, v);
182 
183     // use case 1: both kvs in kvset
184     this.memstore.add(kv1.clone());
185     this.memstore.add(kv2.clone());
186     verifyScanAcrossSnapshot2(kv1, kv2);
187 
188     // use case 2: both kvs in snapshot
189     this.memstore.snapshot();
190     verifyScanAcrossSnapshot2(kv1, kv2);
191 
192     // use case 3: first in snapshot second in kvset
193     this.memstore = new MemStore();
194     this.memstore.add(kv1.clone());
195     this.memstore.snapshot();
196     this.memstore.add(kv2.clone());
197     verifyScanAcrossSnapshot2(kv1, kv2);
198   }
199 
200   private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
201       throws IOException {
202     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
203     List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
204     assertEquals(1, memstorescanners.size());
205     final KeyValueScanner scanner = memstorescanners.get(0);
206     scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
207     assertEquals(kv1, scanner.next());
208     assertEquals(kv2, scanner.next());
209     assertNull(scanner.next());
210   }
211 
212   private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
213       throws IOException {
214     scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
215     List<KeyValue> returned = Lists.newArrayList();
216 
217     while (true) {
218       KeyValue next = scanner.next();
219       if (next == null) break;
220       returned.add(next);
221     }
222 
223     assertTrue(
224         "Got:\n" + Joiner.on("\n").join(returned) +
225         "\nExpected:\n" + Joiner.on("\n").join(expected),
226         Iterables.elementsEqual(Arrays.asList(expected), returned));
227     assertNull(scanner.peek());
228   }
229 
230   public void testMemstoreConcurrentControl() throws IOException {
231     final byte[] row = Bytes.toBytes(1);
232     final byte[] f = Bytes.toBytes("family");
233     final byte[] q1 = Bytes.toBytes("q1");
234     final byte[] q2 = Bytes.toBytes("q2");
235     final byte[] v = Bytes.toBytes("value");
236 
237     MultiVersionConsistencyControl.WriteEntry w =
238         mvcc.beginMemstoreInsert();
239 
240     KeyValue kv1 = new KeyValue(row, f, q1, v);
241     kv1.setMvccVersion(w.getWriteNumber());
242     memstore.add(kv1);
243 
244     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
245     KeyValueScanner s = this.memstore.getScanners().get(0);
246     assertScannerResults(s, new KeyValue[]{});
247 
248     mvcc.completeMemstoreInsert(w);
249 
250     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
251     s = this.memstore.getScanners().get(0);
252     assertScannerResults(s, new KeyValue[]{kv1});
253 
254     w = mvcc.beginMemstoreInsert();
255     KeyValue kv2 = new KeyValue(row, f, q2, v);
256     kv2.setMvccVersion(w.getWriteNumber());
257     memstore.add(kv2);
258 
259     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
260     s = this.memstore.getScanners().get(0);
261     assertScannerResults(s, new KeyValue[]{kv1});
262 
263     mvcc.completeMemstoreInsert(w);
264 
265     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
266     s = this.memstore.getScanners().get(0);
267     assertScannerResults(s, new KeyValue[]{kv1, kv2});
268   }
269 
270   /**
271    * Regression test for HBASE-2616, HBASE-2670.
272    * When we insert a higher-memstoreTS version of a cell but with
273    * the same timestamp, we still need to provide consistent reads
274    * for the same scanner.
275    */
276   public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
277     final byte[] row = Bytes.toBytes(1);
278     final byte[] f = Bytes.toBytes("family");
279     final byte[] q1 = Bytes.toBytes("q1");
280     final byte[] q2 = Bytes.toBytes("q2");
281     final byte[] v1 = Bytes.toBytes("value1");
282     final byte[] v2 = Bytes.toBytes("value2");
283 
284     // INSERT 1: Write both columns val1
285     MultiVersionConsistencyControl.WriteEntry w =
286         mvcc.beginMemstoreInsert();
287 
288     KeyValue kv11 = new KeyValue(row, f, q1, v1);
289     kv11.setMvccVersion(w.getWriteNumber());
290     memstore.add(kv11);
291 
292     KeyValue kv12 = new KeyValue(row, f, q2, v1);
293     kv12.setMvccVersion(w.getWriteNumber());
294     memstore.add(kv12);
295     mvcc.completeMemstoreInsert(w);
296 
297     // BEFORE STARTING INSERT 2, SEE FIRST KVS
298     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
299     KeyValueScanner s = this.memstore.getScanners().get(0);
300     assertScannerResults(s, new KeyValue[]{kv11, kv12});
301 
302     // START INSERT 2: Write both columns val2
303     w = mvcc.beginMemstoreInsert();
304     KeyValue kv21 = new KeyValue(row, f, q1, v2);
305     kv21.setMvccVersion(w.getWriteNumber());
306     memstore.add(kv21);
307 
308     KeyValue kv22 = new KeyValue(row, f, q2, v2);
309     kv22.setMvccVersion(w.getWriteNumber());
310     memstore.add(kv22);
311 
312     // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
313     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
314     s = this.memstore.getScanners().get(0);
315     assertScannerResults(s, new KeyValue[]{kv11, kv12});
316 
317     // COMPLETE INSERT 2
318     mvcc.completeMemstoreInsert(w);
319 
320     // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
321     // See HBASE-1485 for discussion about what we should do with
322     // the duplicate-TS inserts
323     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
324     s = this.memstore.getScanners().get(0);
325     assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
326   }
327 
328   /**
329    * When we insert a higher-memstoreTS deletion of a cell but with
330    * the same timestamp, we still need to provide consistent reads
331    * for the same scanner.
332    */
333   public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
334     final byte[] row = Bytes.toBytes(1);
335     final byte[] f = Bytes.toBytes("family");
336     final byte[] q1 = Bytes.toBytes("q1");
337     final byte[] q2 = Bytes.toBytes("q2");
338     final byte[] v1 = Bytes.toBytes("value1");
339     // INSERT 1: Write both columns val1
340     MultiVersionConsistencyControl.WriteEntry w =
341         mvcc.beginMemstoreInsert();
342 
343     KeyValue kv11 = new KeyValue(row, f, q1, v1);
344     kv11.setMvccVersion(w.getWriteNumber());
345     memstore.add(kv11);
346 
347     KeyValue kv12 = new KeyValue(row, f, q2, v1);
348     kv12.setMvccVersion(w.getWriteNumber());
349     memstore.add(kv12);
350     mvcc.completeMemstoreInsert(w);
351 
352     // BEFORE STARTING INSERT 2, SEE FIRST KVS
353     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
354     KeyValueScanner s = this.memstore.getScanners().get(0);
355     assertScannerResults(s, new KeyValue[]{kv11, kv12});
356 
357     // START DELETE: Insert delete for one of the columns
358     w = mvcc.beginMemstoreInsert();
359     KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
360         KeyValue.Type.DeleteColumn);
361     kvDel.setMvccVersion(w.getWriteNumber());
362     memstore.add(kvDel);
363 
364     // BEFORE COMPLETING DELETE, SEE FIRST KVS
365     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
366     s = this.memstore.getScanners().get(0);
367     assertScannerResults(s, new KeyValue[]{kv11, kv12});
368 
369     // COMPLETE DELETE
370     mvcc.completeMemstoreInsert(w);
371 
372     // NOW WE SHOULD SEE DELETE
373     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
374     s = this.memstore.getScanners().get(0);
375     assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
376   }
377 
378 
379   private static class ReadOwnWritesTester extends Thread {
380     static final int NUM_TRIES = 1000;
381 
382     final byte[] row;
383 
384     final byte[] f = Bytes.toBytes("family");
385     final byte[] q1 = Bytes.toBytes("q1");
386 
387     final MultiVersionConsistencyControl mvcc;
388     final MemStore memstore;
389 
390     AtomicReference<Throwable> caughtException;
391 
392 
393     public ReadOwnWritesTester(int id,
394                                MemStore memstore,
395                                MultiVersionConsistencyControl mvcc,
396                                AtomicReference<Throwable> caughtException)
397     {
398       this.mvcc = mvcc;
399       this.memstore = memstore;
400       this.caughtException = caughtException;
401       row = Bytes.toBytes(id);
402     }
403 
404     public void run() {
405       try {
406         internalRun();
407       } catch (Throwable t) {
408         caughtException.compareAndSet(null, t);
409       }
410     }
411 
412     private void internalRun() throws IOException {
413       for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
414         MultiVersionConsistencyControl.WriteEntry w =
415           mvcc.beginMemstoreInsert();
416 
417         // Insert the sequence value (i)
418         byte[] v = Bytes.toBytes(i);
419 
420         KeyValue kv = new KeyValue(row, f, q1, i, v);
421         kv.setMvccVersion(w.getWriteNumber());
422         memstore.add(kv);
423         mvcc.completeMemstoreInsert(w);
424 
425         // Assert that we can read back
426         MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
427 
428         KeyValueScanner s = this.memstore.getScanners().get(0);
429         s.seek(kv);
430 
431         KeyValue ret = s.next();
432         assertNotNull("Didnt find own write at all", ret);
433         assertEquals("Didnt read own writes",
434                      kv.getTimestamp(), ret.getTimestamp());
435       }
436     }
437   }
438 
439   public void testReadOwnWritesUnderConcurrency() throws Throwable {
440 
441     int NUM_THREADS = 8;
442 
443     ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
444     AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
445 
446     for (int i = 0; i < NUM_THREADS; i++) {
447       threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
448       threads[i].start();
449     }
450 
451     for (int i = 0; i < NUM_THREADS; i++) {
452       threads[i].join();
453     }
454 
455     if (caught.get() != null) {
456       throw caught.get();
457     }
458   }
459 
460   /**
461    * Test memstore snapshots
462    * @throws IOException
463    */
464   public void testSnapshotting() throws IOException {
465     final int snapshotCount = 5;
466     // Add some rows, run a snapshot. Do it a few times.
467     for (int i = 0; i < snapshotCount; i++) {
468       addRows(this.memstore);
469       runSnapshot(this.memstore);
470       KeyValueSkipListSet ss = this.memstore.getSnapshot();
471       assertEquals("History not being cleared", 0, ss.size());
472     }
473   }
474 
475   public void testMultipleVersionsSimple() throws Exception {
476     MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
477     byte [] row = Bytes.toBytes("testRow");
478     byte [] family = Bytes.toBytes("testFamily");
479     byte [] qf = Bytes.toBytes("testQualifier");
480     long [] stamps = {1,2,3};
481     byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
482         Bytes.toBytes("value2")};
483     KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
484     KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
485     KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
486 
487     m.add(key0);
488     m.add(key1);
489     m.add(key2);
490 
491     assertTrue("Expected memstore to hold 3 values, actually has " +
492         m.kvset.size(), m.kvset.size() == 3);
493   }
494 
495   //////////////////////////////////////////////////////////////////////////////
496   // Get tests
497   //////////////////////////////////////////////////////////////////////////////
498 
499   /** Test getNextRow from memstore
500    * @throws InterruptedException
501    */
502   public void testGetNextRow() throws Exception {
503     MultiVersionConsistencyControl.resetThreadReadPoint();
504     addRows(this.memstore);
505     // Add more versions to make it a little more interesting.
506     Thread.sleep(1);
507     addRows(this.memstore);
508     KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
509     assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
510       new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
511     for (int i = 0; i < ROW_COUNT; i++) {
512       KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
513         System.currentTimeMillis()));
514       if (i + 1 == ROW_COUNT) {
515         assertEquals(nr, null);
516       } else {
517         assertTrue(KeyValue.COMPARATOR.compareRows(nr,
518           new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
519       }
520     }
521     //starting from each row, validate results should contain the starting row
522     for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
523       ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false,
524           0, this.memstore.comparator);
525       ScanType scanType = ScanType.USER_SCAN;
526       InternalScanner scanner = new StoreScanner(new Scan(
527           Bytes.toBytes(startRowId)), scanInfo, scanType, null,
528           memstore.getScanners());
529       List<KeyValue> results = new ArrayList<KeyValue>();
530       for (int i = 0; scanner.next(results); i++) {
531         int rowId = startRowId + i;
532         assertTrue("Row name",
533           KeyValue.COMPARATOR.compareRows(results.get(0),
534           Bytes.toBytes(rowId)) == 0);
535         assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
536         List<KeyValue> row = new ArrayList<KeyValue>();
537         for (KeyValue kv : results) {
538           row.add(kv);
539         }
540         isExpectedRowWithoutTimestamps(rowId, row);
541         // Clear out set.  Otherwise row results accumulate.
542         results.clear();
543       }
544     }
545   }
546 
547   public void testGet_memstoreAndSnapShot() throws IOException {
548     byte [] row = Bytes.toBytes("testrow");
549     byte [] fam = Bytes.toBytes("testfamily");
550     byte [] qf1 = Bytes.toBytes("testqualifier1");
551     byte [] qf2 = Bytes.toBytes("testqualifier2");
552     byte [] qf3 = Bytes.toBytes("testqualifier3");
553     byte [] qf4 = Bytes.toBytes("testqualifier4");
554     byte [] qf5 = Bytes.toBytes("testqualifier5");
555     byte [] val = Bytes.toBytes("testval");
556 
557     //Setting up memstore
558     memstore.add(new KeyValue(row, fam ,qf1, val));
559     memstore.add(new KeyValue(row, fam ,qf2, val));
560     memstore.add(new KeyValue(row, fam ,qf3, val));
561     //Creating a snapshot
562     memstore.snapshot();
563     assertEquals(3, memstore.snapshot.size());
564     //Adding value to "new" memstore
565     assertEquals(0, memstore.kvset.size());
566     memstore.add(new KeyValue(row, fam ,qf4, val));
567     memstore.add(new KeyValue(row, fam ,qf5, val));
568     assertEquals(2, memstore.kvset.size());
569   }
570 
571   //////////////////////////////////////////////////////////////////////////////
572   // Delete tests
573   //////////////////////////////////////////////////////////////////////////////
574   public void testGetWithDelete() throws IOException {
575     byte [] row = Bytes.toBytes("testrow");
576     byte [] fam = Bytes.toBytes("testfamily");
577     byte [] qf1 = Bytes.toBytes("testqualifier");
578     byte [] val = Bytes.toBytes("testval");
579 
580     long ts1 = System.nanoTime();
581     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
582     long ts2 = ts1 + 1;
583     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
584     long ts3 = ts2 +1;
585     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
586     memstore.add(put1);
587     memstore.add(put2);
588     memstore.add(put3);
589 
590     assertEquals(3, memstore.kvset.size());
591 
592     KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
593     memstore.delete(del2);
594 
595     List<KeyValue> expected = new ArrayList<KeyValue>();
596     expected.add(put3);
597     expected.add(del2);
598     expected.add(put2);
599     expected.add(put1);
600 
601     assertEquals(4, memstore.kvset.size());
602     int i = 0;
603     for(KeyValue kv : memstore.kvset) {
604       assertEquals(expected.get(i++), kv);
605     }
606   }
607 
608   public void testGetWithDeleteColumn() throws IOException {
609     byte [] row = Bytes.toBytes("testrow");
610     byte [] fam = Bytes.toBytes("testfamily");
611     byte [] qf1 = Bytes.toBytes("testqualifier");
612     byte [] val = Bytes.toBytes("testval");
613 
614     long ts1 = System.nanoTime();
615     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
616     long ts2 = ts1 + 1;
617     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
618     long ts3 = ts2 +1;
619     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
620     memstore.add(put1);
621     memstore.add(put2);
622     memstore.add(put3);
623 
624     assertEquals(3, memstore.kvset.size());
625 
626     KeyValue del2 =
627       new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
628     memstore.delete(del2);
629 
630     List<KeyValue> expected = new ArrayList<KeyValue>();
631     expected.add(put3);
632     expected.add(del2);
633     expected.add(put2);
634     expected.add(put1);
635 
636 
637     assertEquals(4, memstore.kvset.size());
638     int i = 0;
639     for (KeyValue kv: memstore.kvset) {
640       assertEquals(expected.get(i++), kv);
641     }
642   }
643 
644 
645   public void testGetWithDeleteFamily() throws IOException {
646     byte [] row = Bytes.toBytes("testrow");
647     byte [] fam = Bytes.toBytes("testfamily");
648     byte [] qf1 = Bytes.toBytes("testqualifier1");
649     byte [] qf2 = Bytes.toBytes("testqualifier2");
650     byte [] qf3 = Bytes.toBytes("testqualifier3");
651     byte [] val = Bytes.toBytes("testval");
652     long ts = System.nanoTime();
653 
654     KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
655     KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
656     KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
657     KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
658 
659     memstore.add(put1);
660     memstore.add(put2);
661     memstore.add(put3);
662     memstore.add(put4);
663 
664     KeyValue del =
665       new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
666     memstore.delete(del);
667 
668     List<KeyValue> expected = new ArrayList<KeyValue>();
669     expected.add(del);
670     expected.add(put1);
671     expected.add(put2);
672     expected.add(put4);
673     expected.add(put3);
674 
675 
676 
677     assertEquals(5, memstore.kvset.size());
678     int i = 0;
679     for (KeyValue kv: memstore.kvset) {
680       assertEquals(expected.get(i++), kv);
681     }
682   }
683 
684   public void testKeepDeleteInmemstore() {
685     byte [] row = Bytes.toBytes("testrow");
686     byte [] fam = Bytes.toBytes("testfamily");
687     byte [] qf = Bytes.toBytes("testqualifier");
688     byte [] val = Bytes.toBytes("testval");
689     long ts = System.nanoTime();
690     memstore.add(new KeyValue(row, fam, qf, ts, val));
691     KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
692     memstore.delete(delete);
693     assertEquals(2, memstore.kvset.size());
694     assertEquals(delete, memstore.kvset.first());
695   }
696 
697   public void testRetainsDeleteVersion() throws IOException {
698     // add a put to memstore
699     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
700 
701     // now process a specific delete:
702     KeyValue delete = KeyValueTestUtil.create(
703         "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
704     memstore.delete(delete);
705 
706     assertEquals(2, memstore.kvset.size());
707     assertEquals(delete, memstore.kvset.first());
708   }
709   public void testRetainsDeleteColumn() throws IOException {
710     // add a put to memstore
711     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
712 
713     // now process a specific delete:
714     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
715         KeyValue.Type.DeleteColumn, "dont-care");
716     memstore.delete(delete);
717 
718     assertEquals(2, memstore.kvset.size());
719     assertEquals(delete, memstore.kvset.first());
720   }
721   public void testRetainsDeleteFamily() throws IOException {
722     // add a put to memstore
723     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
724 
725     // now process a specific delete:
726     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
727         KeyValue.Type.DeleteFamily, "dont-care");
728     memstore.delete(delete);
729 
730     assertEquals(2, memstore.kvset.size());
731     assertEquals(delete, memstore.kvset.first());
732   }
733 
734   ////////////////////////////////////
735   //Test for timestamps
736   ////////////////////////////////////
737 
738   /**
739    * Test to ensure correctness when using Memstore with multiple timestamps
740    */
741   public void testMultipleTimestamps() throws IOException {
742     long[] timestamps = new long[] {20,10,5,1};
743     Scan scan = new Scan();
744 
745     for (long timestamp: timestamps)
746       addRows(memstore,timestamp);
747 
748     scan.setTimeRange(0, 2);
749     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
750 
751     scan.setTimeRange(20, 82);
752     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
753 
754     scan.setTimeRange(10, 20);
755     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
756 
757     scan.setTimeRange(8, 12);
758     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
759 
760     /*This test is not required for correctness but it should pass when
761      * timestamp range optimization is on*/
762     //scan.setTimeRange(28, 42);
763     //assertTrue(!memstore.shouldSeek(scan));
764   }
765 
766   ////////////////////////////////////
767   //Test for upsert with MSLAB
768   ////////////////////////////////////
769 
770   /**
771    * Test a pathological pattern that shows why we can't currently
772    * use the MSLAB for upsert workloads. This test inserts data
773    * in the following pattern:
774    *
775    * - row0001 through row1000 (fills up one 2M Chunk)
776    * - row0002 through row1001 (fills up another 2M chunk, leaves one reference
777    *   to the first chunk
778    * - row0003 through row1002 (another chunk, another dangling reference)
779    *
780    * This causes OOME pretty quickly if we use MSLAB for upsert
781    * since each 2M chunk is held onto by a single reference.
782    */
783   public void testUpsertMSLAB() throws Exception {
784     Configuration conf = HBaseConfiguration.create();
785     conf.setBoolean(MemStore.USEMSLAB_KEY, true);
786     memstore = new MemStore(conf, KeyValue.COMPARATOR);
787 
788     int ROW_SIZE = 2048;
789     byte[] qualifier = new byte[ROW_SIZE - 4];
790 
791     MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
792     for (int i = 0; i < 3; i++) { System.gc(); }
793     long usageBefore = bean.getHeapMemoryUsage().getUsed();
794 
795     long size = 0;
796     long ts=0;
797 
798     for (int newValue = 0; newValue < 1000; newValue++) {
799       for (int row = newValue; row < newValue + 1000; row++) {
800         byte[] rowBytes = Bytes.toBytes(row);
801         size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
802       }
803     }
804     System.out.println("Wrote " + ts + " vals");
805     for (int i = 0; i < 3; i++) { System.gc(); }
806     long usageAfter = bean.getHeapMemoryUsage().getUsed();
807     System.out.println("Memory used: " + (usageAfter - usageBefore)
808         + " (heapsize: " + memstore.heapSize() +
809         " size: " + size + ")");
810   }
811 
812   //////////////////////////////////////////////////////////////////////////////
813   // Helpers
814   //////////////////////////////////////////////////////////////////////////////
815   private static byte [] makeQualifier(final int i1, final int i2){
816     return Bytes.toBytes(Integer.toString(i1) + ";" +
817         Integer.toString(i2));
818   }
819 
820   /**
821    * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
822    * as older keyvalues are deleted from the memstore.
823    * @throws Exception
824    */
825   public void testUpsertMemstoreSize() throws Exception {
826     Configuration conf = HBaseConfiguration.create();
827     memstore = new MemStore(conf, KeyValue.COMPARATOR);
828     long oldSize = memstore.size.get();
829 
830     List<Cell> l = new ArrayList<Cell>();
831     KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
832     KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
833     KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");
834 
835     kv1.setMvccVersion(1); kv2.setMvccVersion(1);kv3.setMvccVersion(1);
836     l.add(kv1); l.add(kv2); l.add(kv3);
837 
838     this.memstore.upsert(l, 2);// readpoint is 2
839     long newSize = this.memstore.size.get();
840     assert(newSize > oldSize);
841 
842     KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
843     kv4.setMvccVersion(1);
844     l.clear(); l.add(kv4);
845     this.memstore.upsert(l, 3);
846     assertEquals(newSize, this.memstore.size.get());
847     //this.memstore = null;
848   }
849 
850   ////////////////////////////////////
851   // Test for periodic memstore flushes 
852   // based on time of oldest edit
853   ////////////////////////////////////
854 
855   /**
856    * Tests that the timeOfOldestEdit is updated correctly for the 
857    * various edit operations in memstore.
858    * @throws Exception
859    */
860   public void testUpdateToTimeOfOldestEdit() throws Exception {
861     try {
862       EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
863       EnvironmentEdgeManager.injectEdge(edge);
864       MemStore memstore = new MemStore();
865       long t = memstore.timeOfOldestEdit();
866       assertEquals(t, Long.MAX_VALUE);
867 
868       // test the case that the timeOfOldestEdit is updated after a KV add
869       memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
870       t = memstore.timeOfOldestEdit();
871       assertTrue(t == 1234);
872       // snapshot() will reset timeOfOldestEdit. The method will also assert the 
873       // value is reset to Long.MAX_VALUE
874       t = runSnapshot(memstore);
875 
876       // test the case that the timeOfOldestEdit is updated after a KV delete
877       memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
878       t = memstore.timeOfOldestEdit();
879       assertTrue(t == 1234);
880       t = runSnapshot(memstore);
881 
882       // test the case that the timeOfOldestEdit is updated after a KV upsert
883       List<Cell> l = new ArrayList<Cell>();
884       KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
885       kv1.setMvccVersion(100);
886       l.add(kv1);
887       memstore.upsert(l, 1000);
888       t = memstore.timeOfOldestEdit();
889       assertTrue(t == 1234);
890     } finally {
891       EnvironmentEdgeManager.reset();
892     }
893   }
894 
895   /**
896    * Tests the HRegion.shouldFlush method - adds an edit in the memstore
897    * and checks that shouldFlush returns true, and another where it disables
898    * the periodic flush functionality and tests whether shouldFlush returns
899    * false. 
900    * @throws Exception
901    */
902   public void testShouldFlush() throws Exception {
903     Configuration conf = new Configuration();
904     conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
905     checkShouldFlush(conf, true);
906     // test disable flush
907     conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
908     checkShouldFlush(conf, false);
909   }
910 
911   private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
912     try {
913       EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
914       EnvironmentEdgeManager.injectEdge(edge);
915       HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf);
916       HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
917 
918       Map<byte[], Store> stores = region.getStores();
919       assertTrue(stores.size() == 1);
920 
921       Store s = stores.entrySet().iterator().next().getValue();
922       edge.setCurrentTimeMillis(1234);
923       s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
924       edge.setCurrentTimeMillis(1234 + 100);
925       assertTrue(region.shouldFlush() == false);
926       edge.setCurrentTimeMillis(1234 + 10000);
927       assertTrue(region.shouldFlush() == expected);
928     } finally {
929       EnvironmentEdgeManager.reset();
930     }
931   }
932 
933   private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
934     long t = 1234;
935     @Override
936     public long currentTimeMillis() {
937       return t; 
938     }
939     public void setCurrentTimeMillis(long t) {
940       this.t = t;
941     }
942   }
943 
944   /**
945    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
946    * @param hmc Instance to add rows to.
947    * @return How many rows we added.
948    * @throws IOException
949    */
950   private int addRows(final MemStore hmc) {
951     return addRows(hmc, HConstants.LATEST_TIMESTAMP);
952   }
953 
954   /**
955    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
956    * @param hmc Instance to add rows to.
957    * @return How many rows we added.
958    * @throws IOException
959    */
960   private int addRows(final MemStore hmc, final long ts) {
961     for (int i = 0; i < ROW_COUNT; i++) {
962       long timestamp = ts == HConstants.LATEST_TIMESTAMP?
963         System.currentTimeMillis(): ts;
964       for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
965         byte [] row = Bytes.toBytes(i);
966         byte [] qf = makeQualifier(i, ii);
967         hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
968       }
969     }
970     return ROW_COUNT;
971   }
972 
973   private long runSnapshot(final MemStore hmc) throws UnexpectedException {
974     // Save off old state.
975     int oldHistorySize = hmc.getSnapshot().size();
976     hmc.snapshot();
977     KeyValueSkipListSet ss = hmc.getSnapshot();
978     // Make some assertions about what just happened.
979     assertTrue("History size has not increased", oldHistorySize < ss.size());
980     long t = memstore.timeOfOldestEdit();
981     assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
982     hmc.clearSnapshot(ss);
983     return t;
984   }
985 
986   private void isExpectedRowWithoutTimestamps(final int rowIndex,
987       List<KeyValue> kvs) {
988     int i = 0;
989     for (KeyValue kv: kvs) {
990       String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
991       String colnameStr = Bytes.toString(kv.getQualifier());
992       assertEquals("Column name", colnameStr, expectedColname);
993       // Value is column name as bytes.  Usually result is
994       // 100 bytes in size at least. This is the default size
995       // for BytesWriteable.  For comparison, convert bytes to
996       // String and trim to remove trailing null bytes.
997       String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
998         kv.getValueLength());
999       assertEquals("Content", colnameStr, colvalueStr);
1000     }
1001   }
1002 
1003   private KeyValue getDeleteKV(byte [] row) {
1004     return new KeyValue(row, Bytes.toBytes("test_col"), null,
1005       HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
1006   }
1007 
1008   private KeyValue getKV(byte [] row, byte [] value) {
1009     return new KeyValue(row, Bytes.toBytes("test_col"), null,
1010       HConstants.LATEST_TIMESTAMP, value);
1011   }
1012   private static void addRows(int count, final MemStore mem) {
1013     long nanos = System.nanoTime();
1014 
1015     for (int i = 0 ; i < count ; i++) {
1016       if (i % 1000 == 0) {
1017 
1018         System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
1019         nanos = System.nanoTime();
1020       }
1021       long timestamp = System.currentTimeMillis();
1022 
1023       for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
1024         byte [] row = Bytes.toBytes(i);
1025         byte [] qf = makeQualifier(i, ii);
1026         mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
1027       }
1028     }
1029   }
1030 
1031 
1032   static void doScan(MemStore ms, int iteration) throws IOException {
1033     long nanos = System.nanoTime();
1034     KeyValueScanner s = ms.getScanners().get(0);
1035     s.seek(KeyValue.createFirstOnRow(new byte[]{}));
1036 
1037     System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
1038     int cnt=0;
1039     while(s.next() != null) ++cnt;
1040 
1041     System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
1042 
1043   }
1044 
1045   public static void main(String [] args) throws IOException {
1046     MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
1047     MemStore ms = new MemStore();
1048 
1049     long n1 = System.nanoTime();
1050     addRows(25000, ms);
1051     System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
1052 
1053 
1054     System.out.println("foo");
1055 
1056     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
1057 
1058     for (int i = 0 ; i < 50 ; i++)
1059       doScan(ms, i);
1060 
1061   }
1062 
1063 
1064 
1065 }
1066