1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.IOException;
23  import java.lang.management.ManagementFactory;
24  import java.lang.management.MemoryMXBean;
25  import java.rmi.UnexpectedException;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collections;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.atomic.AtomicReference;
32  
33  import junit.framework.TestCase;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.*;
39  import org.apache.hadoop.hbase.client.Scan;
40  import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
41  import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.EnvironmentEdge;
44  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45  
46  import com.google.common.base.Joiner;
47  import com.google.common.collect.Iterables;
48  import com.google.common.collect.Lists;
49  import org.junit.experimental.categories.Category;
50  
51  /** memstore test case */
52  @Category(SmallTests.class)
53  public class TestMemStore extends TestCase {
54    private final Log LOG = LogFactory.getLog(this.getClass());
55    private MemStore memstore;
56    private static final int ROW_COUNT = 10;
57    private static final int QUALIFIER_COUNT = ROW_COUNT;
58    private static final byte [] FAMILY = Bytes.toBytes("column");
59    private static final byte [] CONTENTS = Bytes.toBytes("contents");
60    private static final byte [] BASIC = Bytes.toBytes("basic");
61    private static final String CONTENTSTR = "contentstr";
62    private MultiVersionConsistencyControl mvcc;
63  
64    @Override
65    public void setUp() throws Exception {
66      super.setUp();
67      this.mvcc = new MultiVersionConsistencyControl();
68      this.memstore = new MemStore();
69      SchemaMetrics.setUseTableNameInTest(false);
70    }
71  
72    public void testPutSameKey() {
73      byte [] bytes = Bytes.toBytes(getName());
74      KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
75      this.memstore.add(kv);
76      byte [] other = Bytes.toBytes("somethingelse");
77      KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
78      this.memstore.add(samekey);
79      KeyValue found = this.memstore.kvset.first();
80      assertEquals(1, this.memstore.kvset.size());
81      assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(),
82        found.getValue()));
83    }
84  
85    /**
86     * Test memstore snapshot happening while scanning.
87     * @throws IOException
88     */
89    public void testScanAcrossSnapshot() throws IOException {
90      int rowCount = addRows(this.memstore);
91      List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
92      Scan scan = new Scan();
93      List<KeyValue> result = new ArrayList<KeyValue>();
94      MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
95      ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false,
96          0, this.memstore.comparator);
97      ScanType scanType = ScanType.USER_SCAN;
98      StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
99      int count = 0;
100     try {
101       while (s.next(result)) {
102         LOG.info(result);
103         count++;
104         // Row count is same as column count.
105         assertEquals(rowCount, result.size());
106         result.clear();
107       }
108     } finally {
109       s.close();
110     }
111     assertEquals(rowCount, count);
112     for (KeyValueScanner scanner : memstorescanners) {
113       scanner.close();
114     }
115 
116     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
117     memstorescanners = this.memstore.getScanners();
118     // Now assert can count same number even if a snapshot mid-scan.
119     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
120     count = 0;
121     try {
122       while (s.next(result)) {
123         LOG.info(result);
124         // Assert the stuff is coming out in right order.
125         assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
126         count++;
127         // Row count is same as column count.
128         assertEquals(rowCount, result.size());
129         if (count == 2) {
130           this.memstore.snapshot();
131           LOG.info("Snapshotted");
132         }
133         result.clear();
134       }
135     } finally {
136       s.close();
137     }
138     assertEquals(rowCount, count);
139     for (KeyValueScanner scanner : memstorescanners) {
140       scanner.close();
141     }
142     memstorescanners = this.memstore.getScanners();
143     // Assert that new values are seen in kvset as we scan.
144     long ts = System.currentTimeMillis();
145     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
146     count = 0;
147     int snapshotIndex = 5;
148     try {
149       while (s.next(result)) {
150         LOG.info(result);
151         // Assert the stuff is coming out in right order.
152         assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
153         // Row count is same as column count.
154         assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
155         count++;
156         if (count == snapshotIndex) {
157           this.memstore.snapshot();
158           this.memstore.clearSnapshot(this.memstore.getSnapshot());
159           // Added more rows into kvset.  But the scanner wont see these rows.
160           addRows(this.memstore, ts);
161           LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
162         }
163         result.clear();
164       }
165     } finally {
166       s.close();
167     }
168     assertEquals(rowCount, count);
169   }
170 
171   /**
172    * A simple test which verifies the 3 possible states when scanning across snapshot.
173    * @throws IOException
174    */
175   public void testScanAcrossSnapshot2() throws IOException {
176     // we are going to the scanning across snapshot with two kvs
177     // kv1 should always be returned before kv2
178     final byte[] one = Bytes.toBytes(1);
179     final byte[] two = Bytes.toBytes(2);
180     final byte[] f = Bytes.toBytes("f");
181     final byte[] q = Bytes.toBytes("q");
182     final byte[] v = Bytes.toBytes(3);
183 
184     final KeyValue kv1 = new KeyValue(one, f, q, v);
185     final KeyValue kv2 = new KeyValue(two, f, q, v);
186 
187     // use case 1: both kvs in kvset
188     this.memstore.add(kv1.clone());
189     this.memstore.add(kv2.clone());
190     verifyScanAcrossSnapshot2(kv1, kv2);
191 
192     // use case 2: both kvs in snapshot
193     this.memstore.snapshot();
194     verifyScanAcrossSnapshot2(kv1, kv2);
195 
196     // use case 3: first in snapshot second in kvset
197     this.memstore = new MemStore();
198     this.memstore.add(kv1.clone());
199     this.memstore.snapshot();
200     this.memstore.add(kv2.clone());
201     verifyScanAcrossSnapshot2(kv1, kv2);
202   }
203 
204   private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
205       throws IOException {
206     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
207     List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
208     assertEquals(1, memstorescanners.size());
209     final KeyValueScanner scanner = memstorescanners.get(0);
210     scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
211     assertEquals(kv1, scanner.next());
212     assertEquals(kv2, scanner.next());
213     assertNull(scanner.next());
214   }
215 
216   private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
217       throws IOException {
218     scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
219     List<KeyValue> returned = Lists.newArrayList();
220 
221     while (true) {
222       KeyValue next = scanner.next();
223       if (next == null) break;
224       returned.add(next);
225     }
226 
227     assertTrue(
228         "Got:\n" + Joiner.on("\n").join(returned) +
229         "\nExpected:\n" + Joiner.on("\n").join(expected),
230         Iterables.elementsEqual(Arrays.asList(expected), returned));
231     assertNull(scanner.peek());
232   }
233 
234   public void testMemstoreConcurrentControl() throws IOException {
235     final byte[] row = Bytes.toBytes(1);
236     final byte[] f = Bytes.toBytes("family");
237     final byte[] q1 = Bytes.toBytes("q1");
238     final byte[] q2 = Bytes.toBytes("q2");
239     final byte[] v = Bytes.toBytes("value");
240 
241     MultiVersionConsistencyControl.WriteEntry w =
242         mvcc.beginMemstoreInsert();
243 
244     KeyValue kv1 = new KeyValue(row, f, q1, v);
245     kv1.setMemstoreTS(w.getWriteNumber());
246     memstore.add(kv1);
247 
248     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
249     KeyValueScanner s = this.memstore.getScanners().get(0);
250     assertScannerResults(s, new KeyValue[]{});
251 
252     mvcc.completeMemstoreInsert(w);
253 
254     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
255     s = this.memstore.getScanners().get(0);
256     assertScannerResults(s, new KeyValue[]{kv1});
257 
258     w = mvcc.beginMemstoreInsert();
259     KeyValue kv2 = new KeyValue(row, f, q2, v);
260     kv2.setMemstoreTS(w.getWriteNumber());
261     memstore.add(kv2);
262 
263     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
264     s = this.memstore.getScanners().get(0);
265     assertScannerResults(s, new KeyValue[]{kv1});
266 
267     mvcc.completeMemstoreInsert(w);
268 
269     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
270     s = this.memstore.getScanners().get(0);
271     assertScannerResults(s, new KeyValue[]{kv1, kv2});
272   }
273 
274   /**
275    * Regression test for HBASE-2616, HBASE-2670.
276    * When we insert a higher-memstoreTS version of a cell but with
277    * the same timestamp, we still need to provide consistent reads
278    * for the same scanner.
279    */
280   public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
281     final byte[] row = Bytes.toBytes(1);
282     final byte[] f = Bytes.toBytes("family");
283     final byte[] q1 = Bytes.toBytes("q1");
284     final byte[] q2 = Bytes.toBytes("q2");
285     final byte[] v1 = Bytes.toBytes("value1");
286     final byte[] v2 = Bytes.toBytes("value2");
287 
288     // INSERT 1: Write both columns val1
289     MultiVersionConsistencyControl.WriteEntry w =
290         mvcc.beginMemstoreInsert();
291 
292     KeyValue kv11 = new KeyValue(row, f, q1, v1);
293     kv11.setMemstoreTS(w.getWriteNumber());
294     memstore.add(kv11);
295 
296     KeyValue kv12 = new KeyValue(row, f, q2, v1);
297     kv12.setMemstoreTS(w.getWriteNumber());
298     memstore.add(kv12);
299     mvcc.completeMemstoreInsert(w);
300 
301     // BEFORE STARTING INSERT 2, SEE FIRST KVS
302     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
303     KeyValueScanner s = this.memstore.getScanners().get(0);
304     assertScannerResults(s, new KeyValue[]{kv11, kv12});
305 
306     // START INSERT 2: Write both columns val2
307     w = mvcc.beginMemstoreInsert();
308     KeyValue kv21 = new KeyValue(row, f, q1, v2);
309     kv21.setMemstoreTS(w.getWriteNumber());
310     memstore.add(kv21);
311 
312     KeyValue kv22 = new KeyValue(row, f, q2, v2);
313     kv22.setMemstoreTS(w.getWriteNumber());
314     memstore.add(kv22);
315 
316     // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
317     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
318     s = this.memstore.getScanners().get(0);
319     assertScannerResults(s, new KeyValue[]{kv11, kv12});
320 
321     // COMPLETE INSERT 2
322     mvcc.completeMemstoreInsert(w);
323 
324     // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
325     // See HBASE-1485 for discussion about what we should do with
326     // the duplicate-TS inserts
327     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
328     s = this.memstore.getScanners().get(0);
329     assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
330   }
331 
332   /**
333    * When we insert a higher-memstoreTS deletion of a cell but with
334    * the same timestamp, we still need to provide consistent reads
335    * for the same scanner.
336    */
337   public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
338     final byte[] row = Bytes.toBytes(1);
339     final byte[] f = Bytes.toBytes("family");
340     final byte[] q1 = Bytes.toBytes("q1");
341     final byte[] q2 = Bytes.toBytes("q2");
342     final byte[] v1 = Bytes.toBytes("value1");
343     // INSERT 1: Write both columns val1
344     MultiVersionConsistencyControl.WriteEntry w =
345         mvcc.beginMemstoreInsert();
346 
347     KeyValue kv11 = new KeyValue(row, f, q1, v1);
348     kv11.setMemstoreTS(w.getWriteNumber());
349     memstore.add(kv11);
350 
351     KeyValue kv12 = new KeyValue(row, f, q2, v1);
352     kv12.setMemstoreTS(w.getWriteNumber());
353     memstore.add(kv12);
354     mvcc.completeMemstoreInsert(w);
355 
356     // BEFORE STARTING INSERT 2, SEE FIRST KVS
357     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
358     KeyValueScanner s = this.memstore.getScanners().get(0);
359     assertScannerResults(s, new KeyValue[]{kv11, kv12});
360 
361     // START DELETE: Insert delete for one of the columns
362     w = mvcc.beginMemstoreInsert();
363     KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
364         KeyValue.Type.DeleteColumn);
365     kvDel.setMemstoreTS(w.getWriteNumber());
366     memstore.add(kvDel);
367 
368     // BEFORE COMPLETING DELETE, SEE FIRST KVS
369     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
370     s = this.memstore.getScanners().get(0);
371     assertScannerResults(s, new KeyValue[]{kv11, kv12});
372 
373     // COMPLETE DELETE
374     mvcc.completeMemstoreInsert(w);
375 
376     // NOW WE SHOULD SEE DELETE
377     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
378     s = this.memstore.getScanners().get(0);
379     assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
380   }
381 
382 
383   private static class ReadOwnWritesTester extends Thread {
384     static final int NUM_TRIES = 1000;
385 
386     final byte[] row;
387 
388     final byte[] f = Bytes.toBytes("family");
389     final byte[] q1 = Bytes.toBytes("q1");
390 
391     final MultiVersionConsistencyControl mvcc;
392     final MemStore memstore;
393 
394     AtomicReference<Throwable> caughtException;
395 
396 
397     public ReadOwnWritesTester(int id,
398                                MemStore memstore,
399                                MultiVersionConsistencyControl mvcc,
400                                AtomicReference<Throwable> caughtException)
401     {
402       this.mvcc = mvcc;
403       this.memstore = memstore;
404       this.caughtException = caughtException;
405       row = Bytes.toBytes(id);
406     }
407 
408     public void run() {
409       try {
410         internalRun();
411       } catch (Throwable t) {
412         caughtException.compareAndSet(null, t);
413       }
414     }
415 
416     private void internalRun() throws IOException {
417       for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
418         MultiVersionConsistencyControl.WriteEntry w =
419           mvcc.beginMemstoreInsert();
420 
421         // Insert the sequence value (i)
422         byte[] v = Bytes.toBytes(i);
423 
424         KeyValue kv = new KeyValue(row, f, q1, i, v);
425         kv.setMemstoreTS(w.getWriteNumber());
426         memstore.add(kv);
427         mvcc.completeMemstoreInsert(w);
428 
429         // Assert that we can read back
430         MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
431 
432         KeyValueScanner s = this.memstore.getScanners().get(0);
433         s.seek(kv);
434 
435         KeyValue ret = s.next();
436         assertNotNull("Didnt find own write at all", ret);
437         assertEquals("Didnt read own writes",
438                      kv.getTimestamp(), ret.getTimestamp());
439       }
440     }
441   }
442 
443   public void testReadOwnWritesUnderConcurrency() throws Throwable {
444 
445     int NUM_THREADS = 8;
446 
447     ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
448     AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
449 
450     for (int i = 0; i < NUM_THREADS; i++) {
451       threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
452       threads[i].start();
453     }
454 
455     for (int i = 0; i < NUM_THREADS; i++) {
456       threads[i].join();
457     }
458 
459     if (caught.get() != null) {
460       throw caught.get();
461     }
462   }
463 
464   /**
465    * Test memstore snapshots
466    * @throws IOException
467    */
468   public void testSnapshotting() throws IOException {
469     final int snapshotCount = 5;
470     // Add some rows, run a snapshot. Do it a few times.
471     for (int i = 0; i < snapshotCount; i++) {
472       addRows(this.memstore);
473       runSnapshot(this.memstore);
474       KeyValueSkipListSet ss = this.memstore.getSnapshot();
475       assertEquals("History not being cleared", 0, ss.size());
476     }
477   }
478 
479   public void testMultipleVersionsSimple() throws Exception {
480     MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
481     byte [] row = Bytes.toBytes("testRow");
482     byte [] family = Bytes.toBytes("testFamily");
483     byte [] qf = Bytes.toBytes("testQualifier");
484     long [] stamps = {1,2,3};
485     byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
486         Bytes.toBytes("value2")};
487     KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
488     KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
489     KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
490 
491     m.add(key0);
492     m.add(key1);
493     m.add(key2);
494 
495     assertTrue("Expected memstore to hold 3 values, actually has " +
496         m.kvset.size(), m.kvset.size() == 3);
497   }
498 
499   public void testBinary() throws IOException {
500     MemStore mc = new MemStore(new Configuration(), KeyValue.ROOT_COMPARATOR);
501     final int start = 43;
502     final int end = 46;
503     for (int k = start; k <= end; k++) {
504       byte [] kk = Bytes.toBytes(k);
505       byte [] row =
506         Bytes.toBytes(".META.,table," + Bytes.toString(kk) + ",1," + k);
507       KeyValue key = new KeyValue(row, CONTENTS, BASIC,
508         System.currentTimeMillis(),
509         (CONTENTSTR + k).getBytes(HConstants.UTF8_ENCODING));
510       mc.add(key);
511       System.out.println(key);
512 //      key = new KeyValue(row, Bytes.toBytes(ANCHORNUM + k),
513 //        System.currentTimeMillis(),
514 //        (ANCHORSTR + k).getBytes(HConstants.UTF8_ENCODING));
515 //      mc.add(key);
516 //      System.out.println(key);
517     }
518     int index = start;
519     for (KeyValue kv: mc.kvset) {
520       System.out.println(kv);
521       byte [] b = kv.getRow();
522       // Hardcoded offsets into String
523       String str = Bytes.toString(b, 13, 4);
524       byte [] bb = Bytes.toBytes(index);
525       String bbStr = Bytes.toString(bb);
526       assertEquals(str, bbStr);
527       index++;
528     }
529   }
530 
531   //////////////////////////////////////////////////////////////////////////////
532   // Get tests
533   //////////////////////////////////////////////////////////////////////////////
534 
535   /** Test getNextRow from memstore
536    * @throws InterruptedException
537    */
538   public void testGetNextRow() throws Exception {
539     MultiVersionConsistencyControl.resetThreadReadPoint();
540     addRows(this.memstore);
541     // Add more versions to make it a little more interesting.
542     Thread.sleep(1);
543     addRows(this.memstore);
544     KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
545     assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
546       new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
547     for (int i = 0; i < ROW_COUNT; i++) {
548       KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
549         System.currentTimeMillis()));
550       if (i + 1 == ROW_COUNT) {
551         assertEquals(nr, null);
552       } else {
553         assertTrue(KeyValue.COMPARATOR.compareRows(nr,
554           new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
555       }
556     }
557     //starting from each row, validate results should contain the starting row
558     for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
559       ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false,
560           0, this.memstore.comparator);
561       ScanType scanType = ScanType.USER_SCAN;
562       InternalScanner scanner = new StoreScanner(new Scan(
563           Bytes.toBytes(startRowId)), scanInfo, scanType, null,
564           memstore.getScanners());
565       List<KeyValue> results = new ArrayList<KeyValue>();
566       for (int i = 0; scanner.next(results); i++) {
567         int rowId = startRowId + i;
568         assertTrue("Row name",
569           KeyValue.COMPARATOR.compareRows(results.get(0),
570           Bytes.toBytes(rowId)) == 0);
571         assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
572         List<KeyValue> row = new ArrayList<KeyValue>();
573         for (KeyValue kv : results) {
574           row.add(kv);
575         }
576         isExpectedRowWithoutTimestamps(rowId, row);
577         // Clear out set.  Otherwise row results accumulate.
578         results.clear();
579       }
580     }
581   }
582 
583   public void testGet_memstoreAndSnapShot() throws IOException {
584     byte [] row = Bytes.toBytes("testrow");
585     byte [] fam = Bytes.toBytes("testfamily");
586     byte [] qf1 = Bytes.toBytes("testqualifier1");
587     byte [] qf2 = Bytes.toBytes("testqualifier2");
588     byte [] qf3 = Bytes.toBytes("testqualifier3");
589     byte [] qf4 = Bytes.toBytes("testqualifier4");
590     byte [] qf5 = Bytes.toBytes("testqualifier5");
591     byte [] val = Bytes.toBytes("testval");
592 
593     //Setting up memstore
594     memstore.add(new KeyValue(row, fam ,qf1, val));
595     memstore.add(new KeyValue(row, fam ,qf2, val));
596     memstore.add(new KeyValue(row, fam ,qf3, val));
597     //Creating a snapshot
598     memstore.snapshot();
599     assertEquals(3, memstore.snapshot.size());
600     //Adding value to "new" memstore
601     assertEquals(0, memstore.kvset.size());
602     memstore.add(new KeyValue(row, fam ,qf4, val));
603     memstore.add(new KeyValue(row, fam ,qf5, val));
604     assertEquals(2, memstore.kvset.size());
605   }
606 
607   //////////////////////////////////////////////////////////////////////////////
608   // Delete tests
609   //////////////////////////////////////////////////////////////////////////////
610   public void testGetWithDelete() throws IOException {
611     byte [] row = Bytes.toBytes("testrow");
612     byte [] fam = Bytes.toBytes("testfamily");
613     byte [] qf1 = Bytes.toBytes("testqualifier");
614     byte [] val = Bytes.toBytes("testval");
615 
616     long ts1 = System.nanoTime();
617     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
618     long ts2 = ts1 + 1;
619     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
620     long ts3 = ts2 +1;
621     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
622     memstore.add(put1);
623     memstore.add(put2);
624     memstore.add(put3);
625 
626     assertEquals(3, memstore.kvset.size());
627 
628     KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
629     memstore.delete(del2);
630 
631     List<KeyValue> expected = new ArrayList<KeyValue>();
632     expected.add(put3);
633     expected.add(del2);
634     expected.add(put2);
635     expected.add(put1);
636 
637     assertEquals(4, memstore.kvset.size());
638     int i = 0;
639     for(KeyValue kv : memstore.kvset) {
640       assertEquals(expected.get(i++), kv);
641     }
642   }
643 
644   public void testGetWithDeleteColumn() throws IOException {
645     byte [] row = Bytes.toBytes("testrow");
646     byte [] fam = Bytes.toBytes("testfamily");
647     byte [] qf1 = Bytes.toBytes("testqualifier");
648     byte [] val = Bytes.toBytes("testval");
649 
650     long ts1 = System.nanoTime();
651     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
652     long ts2 = ts1 + 1;
653     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
654     long ts3 = ts2 +1;
655     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
656     memstore.add(put1);
657     memstore.add(put2);
658     memstore.add(put3);
659 
660     assertEquals(3, memstore.kvset.size());
661 
662     KeyValue del2 =
663       new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
664     memstore.delete(del2);
665 
666     List<KeyValue> expected = new ArrayList<KeyValue>();
667     expected.add(put3);
668     expected.add(del2);
669     expected.add(put2);
670     expected.add(put1);
671 
672 
673     assertEquals(4, memstore.kvset.size());
674     int i = 0;
675     for (KeyValue kv: memstore.kvset) {
676       assertEquals(expected.get(i++), kv);
677     }
678   }
679 
680 
681   public void testGetWithDeleteFamily() throws IOException {
682     byte [] row = Bytes.toBytes("testrow");
683     byte [] fam = Bytes.toBytes("testfamily");
684     byte [] qf1 = Bytes.toBytes("testqualifier1");
685     byte [] qf2 = Bytes.toBytes("testqualifier2");
686     byte [] qf3 = Bytes.toBytes("testqualifier3");
687     byte [] val = Bytes.toBytes("testval");
688     long ts = System.nanoTime();
689 
690     KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
691     KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
692     KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
693     KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
694 
695     memstore.add(put1);
696     memstore.add(put2);
697     memstore.add(put3);
698     memstore.add(put4);
699 
700     KeyValue del =
701       new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
702     memstore.delete(del);
703 
704     List<KeyValue> expected = new ArrayList<KeyValue>();
705     expected.add(del);
706     expected.add(put1);
707     expected.add(put2);
708     expected.add(put4);
709     expected.add(put3);
710 
711 
712 
713     assertEquals(5, memstore.kvset.size());
714     int i = 0;
715     for (KeyValue kv: memstore.kvset) {
716       assertEquals(expected.get(i++), kv);
717     }
718   }
719 
720   public void testKeepDeleteInmemstore() {
721     byte [] row = Bytes.toBytes("testrow");
722     byte [] fam = Bytes.toBytes("testfamily");
723     byte [] qf = Bytes.toBytes("testqualifier");
724     byte [] val = Bytes.toBytes("testval");
725     long ts = System.nanoTime();
726     memstore.add(new KeyValue(row, fam, qf, ts, val));
727     KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
728     memstore.delete(delete);
729     assertEquals(2, memstore.kvset.size());
730     assertEquals(delete, memstore.kvset.first());
731   }
732 
733   public void testRetainsDeleteVersion() throws IOException {
734     // add a put to memstore
735     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
736 
737     // now process a specific delete:
738     KeyValue delete = KeyValueTestUtil.create(
739         "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
740     memstore.delete(delete);
741 
742     assertEquals(2, memstore.kvset.size());
743     assertEquals(delete, memstore.kvset.first());
744   }
745   public void testRetainsDeleteColumn() throws IOException {
746     // add a put to memstore
747     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
748 
749     // now process a specific delete:
750     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
751         KeyValue.Type.DeleteColumn, "dont-care");
752     memstore.delete(delete);
753 
754     assertEquals(2, memstore.kvset.size());
755     assertEquals(delete, memstore.kvset.first());
756   }
757   public void testRetainsDeleteFamily() throws IOException {
758     // add a put to memstore
759     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
760 
761     // now process a specific delete:
762     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
763         KeyValue.Type.DeleteFamily, "dont-care");
764     memstore.delete(delete);
765 
766     assertEquals(2, memstore.kvset.size());
767     assertEquals(delete, memstore.kvset.first());
768   }
769 
770   ////////////////////////////////////
771   //Test for timestamps
772   ////////////////////////////////////
773 
774   /**
775    * Test to ensure correctness when using Memstore with multiple timestamps
776    */
777   public void testMultipleTimestamps() throws IOException {
778     long[] timestamps = new long[] {20,10,5,1};
779     Scan scan = new Scan();
780 
781     for (long timestamp: timestamps)
782       addRows(memstore,timestamp);
783 
784     scan.setTimeRange(0, 2);
785     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
786 
787     scan.setTimeRange(20, 82);
788     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
789 
790     scan.setTimeRange(10, 20);
791     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
792 
793     scan.setTimeRange(8, 12);
794     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
795 
796     /*This test is not required for correctness but it should pass when
797      * timestamp range optimization is on*/
798     //scan.setTimeRange(28, 42);
799     //assertTrue(!memstore.shouldSeek(scan));
800   }
801 
802   ////////////////////////////////////
803   //Test for upsert with MSLAB
804   ////////////////////////////////////
805   
806   /**
807    * Test a pathological pattern that shows why we can't currently
808    * use the MSLAB for upsert workloads. This test inserts data
809    * in the following pattern:
810    * 
811    * - row0001 through row1000 (fills up one 2M Chunk)
812    * - row0002 through row1001 (fills up another 2M chunk, leaves one reference
813    *   to the first chunk
814    * - row0003 through row1002 (another chunk, another dangling reference)
815    * 
816    * This causes OOME pretty quickly if we use MSLAB for upsert
817    * since each 2M chunk is held onto by a single reference.
818    */
819   public void testUpsertMSLAB() throws Exception {
820     Configuration conf = HBaseConfiguration.create();
821     conf.setBoolean(MemStore.USEMSLAB_KEY, true);
822     memstore = new MemStore(conf, KeyValue.COMPARATOR);
823     
824     int ROW_SIZE = 2048;
825     byte[] qualifier = new byte[ROW_SIZE - 4];
826     
827     MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
828     for (int i = 0; i < 3; i++) { System.gc(); }
829     long usageBefore = bean.getHeapMemoryUsage().getUsed();
830     
831     long size = 0;
832     long ts=0;
833     
834     for (int newValue = 0; newValue < 1000; newValue++) {
835       for (int row = newValue; row < newValue + 1000; row++) {
836         byte[] rowBytes = Bytes.toBytes(row);
837         size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
838       }
839     }
840     System.out.println("Wrote " + ts + " vals");
841     for (int i = 0; i < 3; i++) { System.gc(); }
842     long usageAfter = bean.getHeapMemoryUsage().getUsed();
843     System.out.println("Memory used: " + (usageAfter - usageBefore)
844         + " (heapsize: " + memstore.heapSize() + 
845         " size: " + size + ")");
846   }
847   
848   //////////////////////////////////////////////////////////////////////////////
849   // Helpers
850   //////////////////////////////////////////////////////////////////////////////
851   private static byte [] makeQualifier(final int i1, final int i2){
852     return Bytes.toBytes(Integer.toString(i1) + ";" +
853         Integer.toString(i2));
854   }
855 
856   /**
857    * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
858    * as older keyvalues are deleted from the memstore.
859    * @throws Exception
860    */
861   public void testUpsertMemstoreSize() throws Exception {
862     Configuration conf = HBaseConfiguration.create();
863     memstore = new MemStore(conf, KeyValue.COMPARATOR);
864     long oldSize = memstore.size.get();
865     
866     KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
867     this.memstore.upsert(Collections.singletonList(kv1));
868     long newSize = this.memstore.size.get();
869     assert(newSize > oldSize);
870     
871     KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
872     this.memstore.upsert(Collections.singletonList(kv2));
873     assertEquals(newSize, this.memstore.size.get());
874   }
875 
876   ////////////////////////////////////
877   // Test for periodic memstore flushes 
878   // based on time of oldest edit
879   ////////////////////////////////////
880 
881   /**
882    * Tests that the timeOfOldestEdit is updated correctly for the 
883    * various edit operations in memstore.
884    * @throws Exception
885    */
886   public void testUpdateToTimeOfOldestEdit() throws Exception {
887     try {
888       EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
889       EnvironmentEdgeManager.injectEdge(edge);
890       MemStore memstore = new MemStore();
891       long t = memstore.timeOfOldestEdit();
892       assertEquals(t, Long.MAX_VALUE);
893 
894       // test the case that the timeOfOldestEdit is updated after a KV add
895       memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
896       t = memstore.timeOfOldestEdit();
897       assertTrue(t == 1234);
898       // snapshot() will reset timeOfOldestEdit. The method will also assert the 
899       // value is reset to Long.MAX_VALUE
900       t = runSnapshot(memstore);
901 
902       // test the case that the timeOfOldestEdit is updated after a KV delete
903       memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
904       t = memstore.timeOfOldestEdit();
905       assertTrue(t == 1234);
906       t = runSnapshot(memstore);
907 
908       // test the case that the timeOfOldestEdit is updated after a KV upsert
909       List<KeyValue> l = new ArrayList<KeyValue>();
910       KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
911       l.add(kv1);
912       memstore.upsert(l);
913       t = memstore.timeOfOldestEdit();
914       assertTrue(t == 1234);
915     } finally {
916       EnvironmentEdgeManager.reset();
917     }
918   }
919 
920   /**
921    * Tests the HRegion.shouldFlush method - adds an edit in the memstore
922    * and checks that shouldFlush returns true, and another where it disables
923    * the periodic flush functionality and tests whether shouldFlush returns
924    * false. 
925    * @throws Exception
926    */
927   public void testShouldFlush() throws Exception {
928     Configuration conf = new Configuration();
929     conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
930     checkShouldFlush(conf, true);
931     // test disable flush
932     conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
933     checkShouldFlush(conf, false);
934   }
935 
936   private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
937     try {
938       EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
939       EnvironmentEdgeManager.injectEdge(edge);
940       HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf);
941       HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
942 
943       Map<byte[], Store> stores = region.getStores();
944       assertTrue(stores.size() == 1);
945 
946       Store s = stores.entrySet().iterator().next().getValue();
947       edge.setCurrentTimeMillis(1234);
948       s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
949       edge.setCurrentTimeMillis(1234 + 100);
950       assertTrue(region.shouldFlush() == false);
951       edge.setCurrentTimeMillis(1234 + 10000);
952       assertTrue(region.shouldFlush() == expected);
953     } finally {
954       EnvironmentEdgeManager.reset();
955     }
956   }
957 
958   private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
959     long t = 1234;
960     @Override
961     public long currentTimeMillis() {
962       return t; 
963     }
964     public void setCurrentTimeMillis(long t) {
965       this.t = t;
966     }
967   }
968 
969   /**   * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
970    * @param hmc Instance to add rows to.
971    * @return How many rows we added.
972    * @throws IOException
973    */
974   private int addRows(final MemStore hmc) {
975     return addRows(hmc, HConstants.LATEST_TIMESTAMP);
976   }
977 
978   /**
979    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
980    * @param hmc Instance to add rows to.
981    * @return How many rows we added.
982    * @throws IOException
983    */
984   private int addRows(final MemStore hmc, final long ts) {
985     for (int i = 0; i < ROW_COUNT; i++) {
986       long timestamp = ts == HConstants.LATEST_TIMESTAMP?
987         System.currentTimeMillis(): ts;
988       for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
989         byte [] row = Bytes.toBytes(i);
990         byte [] qf = makeQualifier(i, ii);
991         hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
992       }
993     }
994     return ROW_COUNT;
995   }
996 
997   private long runSnapshot(final MemStore hmc) throws UnexpectedException {
998     // Save off old state.
999     int oldHistorySize = hmc.getSnapshot().size();
1000     hmc.snapshot();
1001     KeyValueSkipListSet ss = hmc.getSnapshot();
1002     // Make some assertions about what just happened.
1003     assertTrue("History size has not increased", oldHistorySize < ss.size());
1004     long t = memstore.timeOfOldestEdit();
1005     assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
1006     hmc.clearSnapshot(ss);
1007     return t;
1008   }
1009 
1010   private void isExpectedRowWithoutTimestamps(final int rowIndex,
1011       List<KeyValue> kvs) {
1012     int i = 0;
1013     for (KeyValue kv: kvs) {
1014       String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
1015       String colnameStr = Bytes.toString(kv.getQualifier());
1016       assertEquals("Column name", colnameStr, expectedColname);
1017       // Value is column name as bytes.  Usually result is
1018       // 100 bytes in size at least. This is the default size
1019       // for BytesWriteable.  For comparison, convert bytes to
1020       // String and trim to remove trailing null bytes.
1021       String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
1022         kv.getValueLength());
1023       assertEquals("Content", colnameStr, colvalueStr);
1024     }
1025   }
1026 
1027   private KeyValue getDeleteKV(byte [] row) {
1028     return new KeyValue(row, Bytes.toBytes("test_col"), null,
1029       HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
1030   }
1031 
1032   private KeyValue getKV(byte [] row, byte [] value) {
1033     return new KeyValue(row, Bytes.toBytes("test_col"), null,
1034       HConstants.LATEST_TIMESTAMP, value);
1035   }
1036   private static void addRows(int count, final MemStore mem) {
1037     long nanos = System.nanoTime();
1038 
1039     for (int i = 0 ; i < count ; i++) {
1040       if (i % 1000 == 0) {
1041 
1042         System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
1043         nanos = System.nanoTime();
1044       }
1045       long timestamp = System.currentTimeMillis();
1046 
1047       for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
1048         byte [] row = Bytes.toBytes(i);
1049         byte [] qf = makeQualifier(i, ii);
1050         mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
1051       }
1052     }
1053   }
1054 
1055 
1056   static void doScan(MemStore ms, int iteration) throws IOException {
1057     long nanos = System.nanoTime();
1058     KeyValueScanner s = ms.getScanners().get(0);
1059     s.seek(KeyValue.createFirstOnRow(new byte[]{}));
1060 
1061     System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
1062     int cnt=0;
1063     while(s.next() != null) ++cnt;
1064 
1065     System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
1066 
1067   }
1068 
1069   public static void main(String [] args) throws IOException {
1070     MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
1071     MemStore ms = new MemStore();
1072 
1073     long n1 = System.nanoTime();
1074     addRows(25000, ms);
1075     System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
1076 
1077 
1078     System.out.println("foo");
1079 
1080     MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
1081 
1082     for (int i = 0 ; i < 50 ; i++)
1083       doScan(ms, i);
1084 
1085   }
1086 
1087 
1088 
1089   @org.junit.Rule
1090   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1091     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1092 }
1093