1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.lang.management.ManagementFactory;
24  import java.lang.management.MemoryMXBean;
25  import java.rmi.UnexpectedException;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collections;
29  import java.util.List;
30  import java.util.concurrent.atomic.AtomicReference;
31  
32  import junit.framework.TestCase;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.*;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
40  import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
41  import org.apache.hadoop.hbase.util.Bytes;
42  
43  import com.google.common.base.Joiner;
44  import com.google.common.collect.Iterables;
45  import com.google.common.collect.Lists;
46  import org.junit.experimental.categories.Category;
47  
48  /** memstore test case */
49  @Category(SmallTests.class)
50  public class TestMemStore extends TestCase {
51    private final Log LOG = LogFactory.getLog(this.getClass());
52    private MemStore memstore;
53    private static final int ROW_COUNT = 10;
54    private static final int QUALIFIER_COUNT = ROW_COUNT;
55    private static final byte [] FAMILY = Bytes.toBytes("column");
56    private static final byte [] CONTENTS = Bytes.toBytes("contents");
57    private static final byte [] BASIC = Bytes.toBytes("basic");
58    private static final String CONTENTSTR = "contentstr";
59    private MultiVersionConsistencyControl mvcc;
60  
61    @Override
62    public void setUp() throws Exception {
63      super.setUp();
64      this.mvcc = new MultiVersionConsistencyControl();
65      this.memstore = new MemStore();
66      SchemaMetrics.setUseTableNameInTest(false);
67    }
68  
69    public void testPutSameKey() {
70      byte [] bytes = Bytes.toBytes(getName());
71      KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
72      this.memstore.add(kv);
73      byte [] other = Bytes.toBytes("somethingelse");
74      KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
75      this.memstore.add(samekey);
76      KeyValue found = this.memstore.kvset.first();
77      assertEquals(1, this.memstore.kvset.size());
78      assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(),
79        found.getValue()));
80    }
81  
82    /**
83     * Test memstore snapshot happening while scanning.
84     * @throws IOException
85     */
86    public void testScanAcrossSnapshot() throws IOException {
87      int rowCount = addRows(this.memstore);
88      List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
89      Scan scan = new Scan();
90      List<KeyValue> result = new ArrayList<KeyValue>();
91      MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
92      ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false,
93          0, this.memstore.comparator);
94      ScanType scanType = ScanType.USER_SCAN;
95      StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
96      int count = 0;
97      try {
98        while (s.next(result)) {
99          LOG.info(result);
100         count++;
101         // Row count is same as column count.
102         assertEquals(rowCount, result.size());
103         result.clear();
104       }
105     } finally {
106       s.close();
107     }
108     assertEquals(rowCount, count);
109     for (KeyValueScanner scanner : memstorescanners) {
110       scanner.close();
111     }
112 
113     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
114     memstorescanners = this.memstore.getScanners();
115     // Now assert can count same number even if a snapshot mid-scan.
116     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
117     count = 0;
118     try {
119       while (s.next(result)) {
120         LOG.info(result);
121         // Assert the stuff is coming out in right order.
122         assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
123         count++;
124         // Row count is same as column count.
125         assertEquals(rowCount, result.size());
126         if (count == 2) {
127           this.memstore.snapshot();
128           LOG.info("Snapshotted");
129         }
130         result.clear();
131       }
132     } finally {
133       s.close();
134     }
135     assertEquals(rowCount, count);
136     for (KeyValueScanner scanner : memstorescanners) {
137       scanner.close();
138     }
139     memstorescanners = this.memstore.getScanners();
140     // Assert that new values are seen in kvset as we scan.
141     long ts = System.currentTimeMillis();
142     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
143     count = 0;
144     int snapshotIndex = 5;
145     try {
146       while (s.next(result)) {
147         LOG.info(result);
148         // Assert the stuff is coming out in right order.
149         assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
150         // Row count is same as column count.
151         assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
152         count++;
153         if (count == snapshotIndex) {
154           this.memstore.snapshot();
155           this.memstore.clearSnapshot(this.memstore.getSnapshot());
156           // Added more rows into kvset.  But the scanner wont see these rows.
157           addRows(this.memstore, ts);
158           LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
159         }
160         result.clear();
161       }
162     } finally {
163       s.close();
164     }
165     assertEquals(rowCount, count);
166   }
167 
168   /**
169    * A simple test which verifies the 3 possible states when scanning across snapshot.
170    * @throws IOException
171    */
172   public void testScanAcrossSnapshot2() throws IOException {
173     // we are going to the scanning across snapshot with two kvs
174     // kv1 should always be returned before kv2
175     final byte[] one = Bytes.toBytes(1);
176     final byte[] two = Bytes.toBytes(2);
177     final byte[] f = Bytes.toBytes("f");
178     final byte[] q = Bytes.toBytes("q");
179     final byte[] v = Bytes.toBytes(3);
180 
181     final KeyValue kv1 = new KeyValue(one, f, q, v);
182     final KeyValue kv2 = new KeyValue(two, f, q, v);
183 
184     // use case 1: both kvs in kvset
185     this.memstore.add(kv1.clone());
186     this.memstore.add(kv2.clone());
187     verifyScanAcrossSnapshot2(kv1, kv2);
188 
189     // use case 2: both kvs in snapshot
190     this.memstore.snapshot();
191     verifyScanAcrossSnapshot2(kv1, kv2);
192 
193     // use case 3: first in snapshot second in kvset
194     this.memstore = new MemStore();
195     this.memstore.add(kv1.clone());
196     this.memstore.snapshot();
197     this.memstore.add(kv2.clone());
198     verifyScanAcrossSnapshot2(kv1, kv2);
199   }
200 
201   private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
202       throws IOException {
203     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
204     List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
205     assertEquals(1, memstorescanners.size());
206     final KeyValueScanner scanner = memstorescanners.get(0);
207     scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
208     assertEquals(kv1, scanner.next());
209     assertEquals(kv2, scanner.next());
210     assertNull(scanner.next());
211   }
212 
213   private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
214       throws IOException {
215     scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
216     List<KeyValue> returned = Lists.newArrayList();
217 
218     while (true) {
219       KeyValue next = scanner.next();
220       if (next == null) break;
221       returned.add(next);
222     }
223 
224     assertTrue(
225         "Got:\n" + Joiner.on("\n").join(returned) +
226         "\nExpected:\n" + Joiner.on("\n").join(expected),
227         Iterables.elementsEqual(Arrays.asList(expected), returned));
228     assertNull(scanner.peek());
229   }
230 
231   public void testMemstoreConcurrentControl() throws IOException {
232     final byte[] row = Bytes.toBytes(1);
233     final byte[] f = Bytes.toBytes("family");
234     final byte[] q1 = Bytes.toBytes("q1");
235     final byte[] q2 = Bytes.toBytes("q2");
236     final byte[] v = Bytes.toBytes("value");
237 
238     MultiVersionConsistencyControl.WriteEntry w =
239         mvcc.beginMemstoreInsert();
240 
241     KeyValue kv1 = new KeyValue(row, f, q1, v);
242     kv1.setMemstoreTS(w.getWriteNumber());
243     memstore.add(kv1);
244 
245     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
246     KeyValueScanner s = this.memstore.getScanners().get(0);
247     assertScannerResults(s, new KeyValue[]{});
248 
249     mvcc.completeMemstoreInsert(w);
250 
251     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
252     s = this.memstore.getScanners().get(0);
253     assertScannerResults(s, new KeyValue[]{kv1});
254 
255     w = mvcc.beginMemstoreInsert();
256     KeyValue kv2 = new KeyValue(row, f, q2, v);
257     kv2.setMemstoreTS(w.getWriteNumber());
258     memstore.add(kv2);
259 
260     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
261     s = this.memstore.getScanners().get(0);
262     assertScannerResults(s, new KeyValue[]{kv1});
263 
264     mvcc.completeMemstoreInsert(w);
265 
266     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
267     s = this.memstore.getScanners().get(0);
268     assertScannerResults(s, new KeyValue[]{kv1, kv2});
269   }
270 
271   /**
272    * Regression test for HBASE-2616, HBASE-2670.
273    * When we insert a higher-memstoreTS version of a cell but with
274    * the same timestamp, we still need to provide consistent reads
275    * for the same scanner.
276    */
277   public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
278     final byte[] row = Bytes.toBytes(1);
279     final byte[] f = Bytes.toBytes("family");
280     final byte[] q1 = Bytes.toBytes("q1");
281     final byte[] q2 = Bytes.toBytes("q2");
282     final byte[] v1 = Bytes.toBytes("value1");
283     final byte[] v2 = Bytes.toBytes("value2");
284 
285     // INSERT 1: Write both columns val1
286     MultiVersionConsistencyControl.WriteEntry w =
287         mvcc.beginMemstoreInsert();
288 
289     KeyValue kv11 = new KeyValue(row, f, q1, v1);
290     kv11.setMemstoreTS(w.getWriteNumber());
291     memstore.add(kv11);
292 
293     KeyValue kv12 = new KeyValue(row, f, q2, v1);
294     kv12.setMemstoreTS(w.getWriteNumber());
295     memstore.add(kv12);
296     mvcc.completeMemstoreInsert(w);
297 
298     // BEFORE STARTING INSERT 2, SEE FIRST KVS
299     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
300     KeyValueScanner s = this.memstore.getScanners().get(0);
301     assertScannerResults(s, new KeyValue[]{kv11, kv12});
302 
303     // START INSERT 2: Write both columns val2
304     w = mvcc.beginMemstoreInsert();
305     KeyValue kv21 = new KeyValue(row, f, q1, v2);
306     kv21.setMemstoreTS(w.getWriteNumber());
307     memstore.add(kv21);
308 
309     KeyValue kv22 = new KeyValue(row, f, q2, v2);
310     kv22.setMemstoreTS(w.getWriteNumber());
311     memstore.add(kv22);
312 
313     // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
314     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
315     s = this.memstore.getScanners().get(0);
316     assertScannerResults(s, new KeyValue[]{kv11, kv12});
317 
318     // COMPLETE INSERT 2
319     mvcc.completeMemstoreInsert(w);
320 
321     // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
322     // See HBASE-1485 for discussion about what we should do with
323     // the duplicate-TS inserts
324     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
325     s = this.memstore.getScanners().get(0);
326     assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
327   }
328 
329   /**
330    * When we insert a higher-memstoreTS deletion of a cell but with
331    * the same timestamp, we still need to provide consistent reads
332    * for the same scanner.
333    */
334   public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
335     final byte[] row = Bytes.toBytes(1);
336     final byte[] f = Bytes.toBytes("family");
337     final byte[] q1 = Bytes.toBytes("q1");
338     final byte[] q2 = Bytes.toBytes("q2");
339     final byte[] v1 = Bytes.toBytes("value1");
340     // INSERT 1: Write both columns val1
341     MultiVersionConsistencyControl.WriteEntry w =
342         mvcc.beginMemstoreInsert();
343 
344     KeyValue kv11 = new KeyValue(row, f, q1, v1);
345     kv11.setMemstoreTS(w.getWriteNumber());
346     memstore.add(kv11);
347 
348     KeyValue kv12 = new KeyValue(row, f, q2, v1);
349     kv12.setMemstoreTS(w.getWriteNumber());
350     memstore.add(kv12);
351     mvcc.completeMemstoreInsert(w);
352 
353     // BEFORE STARTING INSERT 2, SEE FIRST KVS
354     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
355     KeyValueScanner s = this.memstore.getScanners().get(0);
356     assertScannerResults(s, new KeyValue[]{kv11, kv12});
357 
358     // START DELETE: Insert delete for one of the columns
359     w = mvcc.beginMemstoreInsert();
360     KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
361         KeyValue.Type.DeleteColumn);
362     kvDel.setMemstoreTS(w.getWriteNumber());
363     memstore.add(kvDel);
364 
365     // BEFORE COMPLETING DELETE, SEE FIRST KVS
366     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
367     s = this.memstore.getScanners().get(0);
368     assertScannerResults(s, new KeyValue[]{kv11, kv12});
369 
370     // COMPLETE DELETE
371     mvcc.completeMemstoreInsert(w);
372 
373     // NOW WE SHOULD SEE DELETE
374     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
375     s = this.memstore.getScanners().get(0);
376     assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
377   }
378 
379 
380   private static class ReadOwnWritesTester extends Thread {
381     static final int NUM_TRIES = 1000;
382 
383     final byte[] row;
384 
385     final byte[] f = Bytes.toBytes("family");
386     final byte[] q1 = Bytes.toBytes("q1");
387 
388     final MultiVersionConsistencyControl mvcc;
389     final MemStore memstore;
390 
391     AtomicReference<Throwable> caughtException;
392 
393 
394     public ReadOwnWritesTester(int id,
395                                MemStore memstore,
396                                MultiVersionConsistencyControl mvcc,
397                                AtomicReference<Throwable> caughtException)
398     {
399       this.mvcc = mvcc;
400       this.memstore = memstore;
401       this.caughtException = caughtException;
402       row = Bytes.toBytes(id);
403     }
404 
405     public void run() {
406       try {
407         internalRun();
408       } catch (Throwable t) {
409         caughtException.compareAndSet(null, t);
410       }
411     }
412 
413     private void internalRun() throws IOException {
414       for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
415         MultiVersionConsistencyControl.WriteEntry w =
416           mvcc.beginMemstoreInsert();
417 
418         // Insert the sequence value (i)
419         byte[] v = Bytes.toBytes(i);
420 
421         KeyValue kv = new KeyValue(row, f, q1, i, v);
422         kv.setMemstoreTS(w.getWriteNumber());
423         memstore.add(kv);
424         mvcc.completeMemstoreInsert(w);
425 
426         // Assert that we can read back
427         MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
428 
429         KeyValueScanner s = this.memstore.getScanners().get(0);
430         s.seek(kv);
431 
432         KeyValue ret = s.next();
433         assertNotNull("Didnt find own write at all", ret);
434         assertEquals("Didnt read own writes",
435                      kv.getTimestamp(), ret.getTimestamp());
436       }
437     }
438   }
439 
440   public void testReadOwnWritesUnderConcurrency() throws Throwable {
441 
442     int NUM_THREADS = 8;
443 
444     ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
445     AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
446 
447     for (int i = 0; i < NUM_THREADS; i++) {
448       threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
449       threads[i].start();
450     }
451 
452     for (int i = 0; i < NUM_THREADS; i++) {
453       threads[i].join();
454     }
455 
456     if (caught.get() != null) {
457       throw caught.get();
458     }
459   }
460 
461   /**
462    * Test memstore snapshots
463    * @throws IOException
464    */
465   public void testSnapshotting() throws IOException {
466     final int snapshotCount = 5;
467     // Add some rows, run a snapshot. Do it a few times.
468     for (int i = 0; i < snapshotCount; i++) {
469       addRows(this.memstore);
470       runSnapshot(this.memstore);
471       KeyValueSkipListSet ss = this.memstore.getSnapshot();
472       assertEquals("History not being cleared", 0, ss.size());
473     }
474   }
475 
476   public void testMultipleVersionsSimple() throws Exception {
477     MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
478     byte [] row = Bytes.toBytes("testRow");
479     byte [] family = Bytes.toBytes("testFamily");
480     byte [] qf = Bytes.toBytes("testQualifier");
481     long [] stamps = {1,2,3};
482     byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
483         Bytes.toBytes("value2")};
484     KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
485     KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
486     KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
487 
488     m.add(key0);
489     m.add(key1);
490     m.add(key2);
491 
492     assertTrue("Expected memstore to hold 3 values, actually has " +
493         m.kvset.size(), m.kvset.size() == 3);
494   }
495 
496   public void testBinary() throws IOException {
497     MemStore mc = new MemStore(new Configuration(), KeyValue.ROOT_COMPARATOR);
498     final int start = 43;
499     final int end = 46;
500     for (int k = start; k <= end; k++) {
501       byte [] kk = Bytes.toBytes(k);
502       byte [] row =
503         Bytes.toBytes(".META.,table," + Bytes.toString(kk) + ",1," + k);
504       KeyValue key = new KeyValue(row, CONTENTS, BASIC,
505         System.currentTimeMillis(),
506         (CONTENTSTR + k).getBytes(HConstants.UTF8_ENCODING));
507       mc.add(key);
508       System.out.println(key);
509 //      key = new KeyValue(row, Bytes.toBytes(ANCHORNUM + k),
510 //        System.currentTimeMillis(),
511 //        (ANCHORSTR + k).getBytes(HConstants.UTF8_ENCODING));
512 //      mc.add(key);
513 //      System.out.println(key);
514     }
515     int index = start;
516     for (KeyValue kv: mc.kvset) {
517       System.out.println(kv);
518       byte [] b = kv.getRow();
519       // Hardcoded offsets into String
520       String str = Bytes.toString(b, 13, 4);
521       byte [] bb = Bytes.toBytes(index);
522       String bbStr = Bytes.toString(bb);
523       assertEquals(str, bbStr);
524       index++;
525     }
526   }
527 
528   //////////////////////////////////////////////////////////////////////////////
529   // Get tests
530   //////////////////////////////////////////////////////////////////////////////
531 
532   /** Test getNextRow from memstore
533    * @throws InterruptedException
534    */
535   public void testGetNextRow() throws Exception {
536     MultiVersionConsistencyControl.resetThreadReadPoint();
537     addRows(this.memstore);
538     // Add more versions to make it a little more interesting.
539     Thread.sleep(1);
540     addRows(this.memstore);
541     KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
542     assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
543       new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
544     for (int i = 0; i < ROW_COUNT; i++) {
545       KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
546         System.currentTimeMillis()));
547       if (i + 1 == ROW_COUNT) {
548         assertEquals(nr, null);
549       } else {
550         assertTrue(KeyValue.COMPARATOR.compareRows(nr,
551           new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
552       }
553     }
554     //starting from each row, validate results should contain the starting row
555     for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
556       ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false,
557           0, this.memstore.comparator);
558       ScanType scanType = ScanType.USER_SCAN;
559       InternalScanner scanner = new StoreScanner(new Scan(
560           Bytes.toBytes(startRowId)), scanInfo, scanType, null,
561           memstore.getScanners());
562       List<KeyValue> results = new ArrayList<KeyValue>();
563       for (int i = 0; scanner.next(results); i++) {
564         int rowId = startRowId + i;
565         assertTrue("Row name",
566           KeyValue.COMPARATOR.compareRows(results.get(0),
567           Bytes.toBytes(rowId)) == 0);
568         assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
569         List<KeyValue> row = new ArrayList<KeyValue>();
570         for (KeyValue kv : results) {
571           row.add(kv);
572         }
573         isExpectedRowWithoutTimestamps(rowId, row);
574         // Clear out set.  Otherwise row results accumulate.
575         results.clear();
576       }
577     }
578   }
579 
580   public void testGet_memstoreAndSnapShot() throws IOException {
581     byte [] row = Bytes.toBytes("testrow");
582     byte [] fam = Bytes.toBytes("testfamily");
583     byte [] qf1 = Bytes.toBytes("testqualifier1");
584     byte [] qf2 = Bytes.toBytes("testqualifier2");
585     byte [] qf3 = Bytes.toBytes("testqualifier3");
586     byte [] qf4 = Bytes.toBytes("testqualifier4");
587     byte [] qf5 = Bytes.toBytes("testqualifier5");
588     byte [] val = Bytes.toBytes("testval");
589 
590     //Setting up memstore
591     memstore.add(new KeyValue(row, fam ,qf1, val));
592     memstore.add(new KeyValue(row, fam ,qf2, val));
593     memstore.add(new KeyValue(row, fam ,qf3, val));
594     //Creating a snapshot
595     memstore.snapshot();
596     assertEquals(3, memstore.snapshot.size());
597     //Adding value to "new" memstore
598     assertEquals(0, memstore.kvset.size());
599     memstore.add(new KeyValue(row, fam ,qf4, val));
600     memstore.add(new KeyValue(row, fam ,qf5, val));
601     assertEquals(2, memstore.kvset.size());
602   }
603 
604   //////////////////////////////////////////////////////////////////////////////
605   // Delete tests
606   //////////////////////////////////////////////////////////////////////////////
607   public void testGetWithDelete() throws IOException {
608     byte [] row = Bytes.toBytes("testrow");
609     byte [] fam = Bytes.toBytes("testfamily");
610     byte [] qf1 = Bytes.toBytes("testqualifier");
611     byte [] val = Bytes.toBytes("testval");
612 
613     long ts1 = System.nanoTime();
614     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
615     long ts2 = ts1 + 1;
616     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
617     long ts3 = ts2 +1;
618     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
619     memstore.add(put1);
620     memstore.add(put2);
621     memstore.add(put3);
622 
623     assertEquals(3, memstore.kvset.size());
624 
625     KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
626     memstore.delete(del2);
627 
628     List<KeyValue> expected = new ArrayList<KeyValue>();
629     expected.add(put3);
630     expected.add(del2);
631     expected.add(put2);
632     expected.add(put1);
633 
634     assertEquals(4, memstore.kvset.size());
635     int i = 0;
636     for(KeyValue kv : memstore.kvset) {
637       assertEquals(expected.get(i++), kv);
638     }
639   }
640 
641   public void testGetWithDeleteColumn() throws IOException {
642     byte [] row = Bytes.toBytes("testrow");
643     byte [] fam = Bytes.toBytes("testfamily");
644     byte [] qf1 = Bytes.toBytes("testqualifier");
645     byte [] val = Bytes.toBytes("testval");
646 
647     long ts1 = System.nanoTime();
648     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
649     long ts2 = ts1 + 1;
650     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
651     long ts3 = ts2 +1;
652     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
653     memstore.add(put1);
654     memstore.add(put2);
655     memstore.add(put3);
656 
657     assertEquals(3, memstore.kvset.size());
658 
659     KeyValue del2 =
660       new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
661     memstore.delete(del2);
662 
663     List<KeyValue> expected = new ArrayList<KeyValue>();
664     expected.add(put3);
665     expected.add(del2);
666     expected.add(put2);
667     expected.add(put1);
668 
669 
670     assertEquals(4, memstore.kvset.size());
671     int i = 0;
672     for (KeyValue kv: memstore.kvset) {
673       assertEquals(expected.get(i++), kv);
674     }
675   }
676 
677 
678   public void testGetWithDeleteFamily() throws IOException {
679     byte [] row = Bytes.toBytes("testrow");
680     byte [] fam = Bytes.toBytes("testfamily");
681     byte [] qf1 = Bytes.toBytes("testqualifier1");
682     byte [] qf2 = Bytes.toBytes("testqualifier2");
683     byte [] qf3 = Bytes.toBytes("testqualifier3");
684     byte [] val = Bytes.toBytes("testval");
685     long ts = System.nanoTime();
686 
687     KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
688     KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
689     KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
690     KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
691 
692     memstore.add(put1);
693     memstore.add(put2);
694     memstore.add(put3);
695     memstore.add(put4);
696 
697     KeyValue del =
698       new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
699     memstore.delete(del);
700 
701     List<KeyValue> expected = new ArrayList<KeyValue>();
702     expected.add(del);
703     expected.add(put1);
704     expected.add(put2);
705     expected.add(put4);
706     expected.add(put3);
707 
708 
709 
710     assertEquals(5, memstore.kvset.size());
711     int i = 0;
712     for (KeyValue kv: memstore.kvset) {
713       assertEquals(expected.get(i++), kv);
714     }
715   }
716 
717   public void testKeepDeleteInmemstore() {
718     byte [] row = Bytes.toBytes("testrow");
719     byte [] fam = Bytes.toBytes("testfamily");
720     byte [] qf = Bytes.toBytes("testqualifier");
721     byte [] val = Bytes.toBytes("testval");
722     long ts = System.nanoTime();
723     memstore.add(new KeyValue(row, fam, qf, ts, val));
724     KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
725     memstore.delete(delete);
726     assertEquals(2, memstore.kvset.size());
727     assertEquals(delete, memstore.kvset.first());
728   }
729 
730   public void testRetainsDeleteVersion() throws IOException {
731     // add a put to memstore
732     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
733 
734     // now process a specific delete:
735     KeyValue delete = KeyValueTestUtil.create(
736         "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
737     memstore.delete(delete);
738 
739     assertEquals(2, memstore.kvset.size());
740     assertEquals(delete, memstore.kvset.first());
741   }
742   public void testRetainsDeleteColumn() throws IOException {
743     // add a put to memstore
744     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
745 
746     // now process a specific delete:
747     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
748         KeyValue.Type.DeleteColumn, "dont-care");
749     memstore.delete(delete);
750 
751     assertEquals(2, memstore.kvset.size());
752     assertEquals(delete, memstore.kvset.first());
753   }
754   public void testRetainsDeleteFamily() throws IOException {
755     // add a put to memstore
756     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
757 
758     // now process a specific delete:
759     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
760         KeyValue.Type.DeleteFamily, "dont-care");
761     memstore.delete(delete);
762 
763     assertEquals(2, memstore.kvset.size());
764     assertEquals(delete, memstore.kvset.first());
765   }
766 
767   ////////////////////////////////////
768   //Test for timestamps
769   ////////////////////////////////////
770 
771   /**
772    * Test to ensure correctness when using Memstore with multiple timestamps
773    */
774   public void testMultipleTimestamps() throws IOException {
775     long[] timestamps = new long[] {20,10,5,1};
776     Scan scan = new Scan();
777 
778     for (long timestamp: timestamps)
779       addRows(memstore,timestamp);
780 
781     scan.setTimeRange(0, 2);
782     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
783 
784     scan.setTimeRange(20, 82);
785     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
786 
787     scan.setTimeRange(10, 20);
788     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
789 
790     scan.setTimeRange(8, 12);
791     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
792 
793     /*This test is not required for correctness but it should pass when
794      * timestamp range optimization is on*/
795     //scan.setTimeRange(28, 42);
796     //assertTrue(!memstore.shouldSeek(scan));
797   }
798 
799   ////////////////////////////////////
800   //Test for upsert with MSLAB
801   ////////////////////////////////////
802   
803   /**
804    * Test a pathological pattern that shows why we can't currently
805    * use the MSLAB for upsert workloads. This test inserts data
806    * in the following pattern:
807    * 
808    * - row0001 through row1000 (fills up one 2M Chunk)
809    * - row0002 through row1001 (fills up another 2M chunk, leaves one reference
810    *   to the first chunk
811    * - row0003 through row1002 (another chunk, another dangling reference)
812    * 
813    * This causes OOME pretty quickly if we use MSLAB for upsert
814    * since each 2M chunk is held onto by a single reference.
815    */
816   public void testUpsertMSLAB() throws Exception {
817     Configuration conf = HBaseConfiguration.create();
818     conf.setBoolean(MemStore.USEMSLAB_KEY, true);
819     memstore = new MemStore(conf, KeyValue.COMPARATOR);
820     
821     int ROW_SIZE = 2048;
822     byte[] qualifier = new byte[ROW_SIZE - 4];
823     
824     MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
825     for (int i = 0; i < 3; i++) { System.gc(); }
826     long usageBefore = bean.getHeapMemoryUsage().getUsed();
827     
828     long size = 0;
829     long ts=0;
830     
831     for (int newValue = 0; newValue < 1000; newValue++) {
832       for (int row = newValue; row < newValue + 1000; row++) {
833         byte[] rowBytes = Bytes.toBytes(row);
834         size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
835       }
836     }
837     System.out.println("Wrote " + ts + " vals");
838     for (int i = 0; i < 3; i++) { System.gc(); }
839     long usageAfter = bean.getHeapMemoryUsage().getUsed();
840     System.out.println("Memory used: " + (usageAfter - usageBefore)
841         + " (heapsize: " + memstore.heapSize() + 
842         " size: " + size + ")");
843   }
844   
845   //////////////////////////////////////////////////////////////////////////////
846   // Helpers
847   //////////////////////////////////////////////////////////////////////////////
848   private static byte [] makeQualifier(final int i1, final int i2){
849     return Bytes.toBytes(Integer.toString(i1) + ";" +
850         Integer.toString(i2));
851   }
852 
853   /**
854    * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
855    * as older keyvalues are deleted from the memstore.
856    * @throws Exception
857    */
858   public void testUpsertMemstoreSize() throws Exception {
859     Configuration conf = HBaseConfiguration.create();
860     memstore = new MemStore(conf, KeyValue.COMPARATOR);
861     long oldSize = memstore.size.get();
862     
863     KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
864     this.memstore.upsert(Collections.singletonList(kv1));
865     long newSize = this.memstore.size.get();
866     assert(newSize > oldSize);
867     
868     KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
869     this.memstore.upsert(Collections.singletonList(kv2));
870     assertEquals(newSize, this.memstore.size.get());
871   }
872 
873   /**   * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
874    * @param hmc Instance to add rows to.
875    * @return How many rows we added.
876    * @throws IOException
877    */
878   private int addRows(final MemStore hmc) {
879     return addRows(hmc, HConstants.LATEST_TIMESTAMP);
880   }
881 
882   /**
883    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
884    * @param hmc Instance to add rows to.
885    * @return How many rows we added.
886    * @throws IOException
887    */
888   private int addRows(final MemStore hmc, final long ts) {
889     for (int i = 0; i < ROW_COUNT; i++) {
890       long timestamp = ts == HConstants.LATEST_TIMESTAMP?
891         System.currentTimeMillis(): ts;
892       for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
893         byte [] row = Bytes.toBytes(i);
894         byte [] qf = makeQualifier(i, ii);
895         hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
896       }
897     }
898     return ROW_COUNT;
899   }
900 
901   private void runSnapshot(final MemStore hmc) throws UnexpectedException {
902     // Save off old state.
903     int oldHistorySize = hmc.getSnapshot().size();
904     hmc.snapshot();
905     KeyValueSkipListSet ss = hmc.getSnapshot();
906     // Make some assertions about what just happened.
907     assertTrue("History size has not increased", oldHistorySize < ss.size());
908     hmc.clearSnapshot(ss);
909   }
910 
911   private void isExpectedRowWithoutTimestamps(final int rowIndex,
912       List<KeyValue> kvs) {
913     int i = 0;
914     for (KeyValue kv: kvs) {
915       String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
916       String colnameStr = Bytes.toString(kv.getQualifier());
917       assertEquals("Column name", colnameStr, expectedColname);
918       // Value is column name as bytes.  Usually result is
919       // 100 bytes in size at least. This is the default size
920       // for BytesWriteable.  For comparison, convert bytes to
921       // String and trim to remove trailing null bytes.
922       String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
923         kv.getValueLength());
924       assertEquals("Content", colnameStr, colvalueStr);
925     }
926   }
927 
928   private KeyValue getDeleteKV(byte [] row) {
929     return new KeyValue(row, Bytes.toBytes("test_col"), null,
930       HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
931   }
932 
933   private KeyValue getKV(byte [] row, byte [] value) {
934     return new KeyValue(row, Bytes.toBytes("test_col"), null,
935       HConstants.LATEST_TIMESTAMP, value);
936   }
937   private static void addRows(int count, final MemStore mem) {
938     long nanos = System.nanoTime();
939 
940     for (int i = 0 ; i < count ; i++) {
941       if (i % 1000 == 0) {
942 
943         System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
944         nanos = System.nanoTime();
945       }
946       long timestamp = System.currentTimeMillis();
947 
948       for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
949         byte [] row = Bytes.toBytes(i);
950         byte [] qf = makeQualifier(i, ii);
951         mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
952       }
953     }
954   }
955 
956 
957   static void doScan(MemStore ms, int iteration) throws IOException {
958     long nanos = System.nanoTime();
959     KeyValueScanner s = ms.getScanners().get(0);
960     s.seek(KeyValue.createFirstOnRow(new byte[]{}));
961 
962     System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
963     int cnt=0;
964     while(s.next() != null) ++cnt;
965 
966     System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
967 
968   }
969 
970   public static void main(String [] args) throws IOException {
971     MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
972     MemStore ms = new MemStore();
973 
974     long n1 = System.nanoTime();
975     addRows(25000, ms);
976     System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
977 
978 
979     System.out.println("foo");
980 
981     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
982 
983     for (int i = 0 ; i < 50 ; i++)
984       doScan(ms, i);
985 
986   }
987 
988 
989 
990   @org.junit.Rule
991   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
992     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
993 }
994