View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.lang.management.ManagementFactory;
23  import java.lang.management.MemoryMXBean;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.concurrent.atomic.AtomicLong;
29  import java.util.concurrent.atomic.AtomicReference;
30  
31  import junit.framework.TestCase;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.Cell;
37  import org.apache.hadoop.hbase.CellUtil;
38  import org.apache.hadoop.hbase.HBaseConfiguration;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.KeepDeletedCells;
43  import org.apache.hadoop.hbase.KeyValue;
44  import org.apache.hadoop.hbase.KeyValueTestUtil;
45  import org.apache.hadoop.hbase.KeyValueUtil;
46  import org.apache.hadoop.hbase.testclassification.MediumTests;
47  import org.apache.hadoop.hbase.client.Scan;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.EnvironmentEdge;
50  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
51  import org.junit.experimental.categories.Category;
52  
53  import com.google.common.base.Joiner;
54  import com.google.common.collect.Iterables;
55  import com.google.common.collect.Lists;
56  
57  /** memstore test case */
58  @Category(MediumTests.class)
59  public class TestDefaultMemStore extends TestCase {
60    private final Log LOG = LogFactory.getLog(this.getClass());
61    private DefaultMemStore memstore;
62    private static final int ROW_COUNT = 10;
63    private static final int QUALIFIER_COUNT = ROW_COUNT;
64    private static final byte [] FAMILY = Bytes.toBytes("column");
65    private MultiVersionConsistencyControl mvcc;
66    private AtomicLong startSeqNum = new AtomicLong(0); 
67  
68    @Override
69    public void setUp() throws Exception {
70      super.setUp();
71      this.mvcc = new MultiVersionConsistencyControl();
72      this.memstore = new DefaultMemStore();
73    }
74  
75    public void testPutSameKey() {
76      byte [] bytes = Bytes.toBytes(getName());
77      KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
78      this.memstore.add(kv);
79      byte [] other = Bytes.toBytes("somethingelse");
80      KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
81      this.memstore.add(samekey);
82      Cell found = this.memstore.cellSet.first();
83      assertEquals(1, this.memstore.cellSet.size());
84      assertTrue(Bytes.toString(found.getValue()), CellUtil.matchingValue(samekey, found));
85    }
86  
87    /**
88     * Test memstore snapshot happening while scanning.
89     * @throws IOException
90     */
91    public void testScanAcrossSnapshot() throws IOException {
92      int rowCount = addRows(this.memstore);
93      List<KeyValueScanner> memstorescanners = this.memstore.getScanners(0);
94      Scan scan = new Scan();
95      List<Cell> result = new ArrayList<Cell>();
96      ScanInfo scanInfo =
97          new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0,
98              this.memstore.comparator);
99      ScanType scanType = ScanType.USER_SCAN;
100     StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
101     int count = 0;
102     try {
103       while (s.next(result)) {
104         LOG.info(result);
105         count++;
106         // Row count is same as column count.
107         assertEquals(rowCount, result.size());
108         result.clear();
109       }
110     } finally {
111       s.close();
112     }
113     assertEquals(rowCount, count);
114     for (KeyValueScanner scanner : memstorescanners) {
115       scanner.close();
116     }
117 
118     memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
119     // Now assert can count same number even if a snapshot mid-scan.
120     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
121     count = 0;
122     try {
123       while (s.next(result)) {
124         LOG.info(result);
125         // Assert the stuff is coming out in right order.
126         assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
127         count++;
128         // Row count is same as column count.
129         assertEquals(rowCount, result.size());
130         if (count == 2) {
131           this.memstore.snapshot();
132           LOG.info("Snapshotted");
133         }
134         result.clear();
135       }
136     } finally {
137       s.close();
138     }
139     assertEquals(rowCount, count);
140     for (KeyValueScanner scanner : memstorescanners) {
141       scanner.close();
142     }
143     memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
144     // Assert that new values are seen in kvset as we scan.
145     long ts = System.currentTimeMillis();
146     s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
147     count = 0;
148     int snapshotIndex = 5;
149     try {
150       while (s.next(result)) {
151         LOG.info(result);
152         // Assert the stuff is coming out in right order.
153         assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
154         // Row count is same as column count.
155         assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
156         count++;
157         if (count == snapshotIndex) {
158           MemStoreSnapshot snapshot = this.memstore.snapshot();
159           this.memstore.clearSnapshot(snapshot.getId());
160           // Added more rows into kvset.  But the scanner wont see these rows.
161           addRows(this.memstore, ts);
162           LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
163         }
164         result.clear();
165       }
166     } finally {
167       s.close();
168     }
169     assertEquals(rowCount, count);
170   }
171 
172   /**
173    * A simple test which verifies the 3 possible states when scanning across snapshot.
174    * @throws IOException
175    * @throws CloneNotSupportedException 
176    */
177   public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
178     // we are going to the scanning across snapshot with two kvs
179     // kv1 should always be returned before kv2
180     final byte[] one = Bytes.toBytes(1);
181     final byte[] two = Bytes.toBytes(2);
182     final byte[] f = Bytes.toBytes("f");
183     final byte[] q = Bytes.toBytes("q");
184     final byte[] v = Bytes.toBytes(3);
185 
186     final KeyValue kv1 = new KeyValue(one, f, q, v);
187     final KeyValue kv2 = new KeyValue(two, f, q, v);
188 
189     // use case 1: both kvs in kvset
190     this.memstore.add(kv1.clone());
191     this.memstore.add(kv2.clone());
192     verifyScanAcrossSnapshot2(kv1, kv2);
193 
194     // use case 2: both kvs in snapshot
195     this.memstore.snapshot();
196     verifyScanAcrossSnapshot2(kv1, kv2);
197 
198     // use case 3: first in snapshot second in kvset
199     this.memstore = new DefaultMemStore();
200     this.memstore.add(kv1.clone());
201     this.memstore.snapshot();
202     this.memstore.add(kv2.clone());
203     verifyScanAcrossSnapshot2(kv1, kv2);
204   }
205 
206   private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
207       throws IOException {
208     List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
209     assertEquals(1, memstorescanners.size());
210     final KeyValueScanner scanner = memstorescanners.get(0);
211     scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
212     assertEquals(kv1, scanner.next());
213     assertEquals(kv2, scanner.next());
214     assertNull(scanner.next());
215   }
216 
217   private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
218       throws IOException {
219     scanner.seek(KeyValueUtil.createFirstOnRow(new byte[]{}));
220     List<Cell> returned = Lists.newArrayList();
221 
222     while (true) {
223       Cell next = scanner.next();
224       if (next == null) break;
225       returned.add(next);
226     }
227 
228     assertTrue(
229         "Got:\n" + Joiner.on("\n").join(returned) +
230         "\nExpected:\n" + Joiner.on("\n").join(expected),
231         Iterables.elementsEqual(Arrays.asList(expected), returned));
232     assertNull(scanner.peek());
233   }
234 
235   public void testMemstoreConcurrentControl() throws IOException {
236     final byte[] row = Bytes.toBytes(1);
237     final byte[] f = Bytes.toBytes("family");
238     final byte[] q1 = Bytes.toBytes("q1");
239     final byte[] q2 = Bytes.toBytes("q2");
240     final byte[] v = Bytes.toBytes("value");
241 
242     MultiVersionConsistencyControl.WriteEntry w =
243         mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
244 
245     KeyValue kv1 = new KeyValue(row, f, q1, v);
246     kv1.setSequenceId(w.getWriteNumber());
247     memstore.add(kv1);
248 
249     KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
250     assertScannerResults(s, new KeyValue[]{});
251 
252     mvcc.completeMemstoreInsert(w);
253 
254     s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
255     assertScannerResults(s, new KeyValue[]{kv1});
256 
257     w = mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
258     KeyValue kv2 = new KeyValue(row, f, q2, v);
259     kv2.setSequenceId(w.getWriteNumber());
260     memstore.add(kv2);
261 
262     s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
263     assertScannerResults(s, new KeyValue[]{kv1});
264 
265     mvcc.completeMemstoreInsert(w);
266 
267     s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
268     assertScannerResults(s, new KeyValue[]{kv1, kv2});
269   }
270 
271   /**
272    * Regression test for HBASE-2616, HBASE-2670.
273    * When we insert a higher-memstoreTS version of a cell but with
274    * the same timestamp, we still need to provide consistent reads
275    * for the same scanner.
276    */
277   public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
278     final byte[] row = Bytes.toBytes(1);
279     final byte[] f = Bytes.toBytes("family");
280     final byte[] q1 = Bytes.toBytes("q1");
281     final byte[] q2 = Bytes.toBytes("q2");
282     final byte[] v1 = Bytes.toBytes("value1");
283     final byte[] v2 = Bytes.toBytes("value2");
284 
285     // INSERT 1: Write both columns val1
286     MultiVersionConsistencyControl.WriteEntry w =
287         mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
288 
289     KeyValue kv11 = new KeyValue(row, f, q1, v1);
290     kv11.setSequenceId(w.getWriteNumber());
291     memstore.add(kv11);
292 
293     KeyValue kv12 = new KeyValue(row, f, q2, v1);
294     kv12.setSequenceId(w.getWriteNumber());
295     memstore.add(kv12);
296     mvcc.completeMemstoreInsert(w);
297 
298     // BEFORE STARTING INSERT 2, SEE FIRST KVS
299     KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
300     assertScannerResults(s, new KeyValue[]{kv11, kv12});
301 
302     // START INSERT 2: Write both columns val2
303     w = mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
304     KeyValue kv21 = new KeyValue(row, f, q1, v2);
305     kv21.setSequenceId(w.getWriteNumber());
306     memstore.add(kv21);
307 
308     KeyValue kv22 = new KeyValue(row, f, q2, v2);
309     kv22.setSequenceId(w.getWriteNumber());
310     memstore.add(kv22);
311 
312     // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
313     s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
314     assertScannerResults(s, new KeyValue[]{kv11, kv12});
315 
316     // COMPLETE INSERT 2
317     mvcc.completeMemstoreInsert(w);
318 
319     // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
320     // See HBASE-1485 for discussion about what we should do with
321     // the duplicate-TS inserts
322     s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
323     assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
324   }
325 
326   /**
327    * When we insert a higher-memstoreTS deletion of a cell but with
328    * the same timestamp, we still need to provide consistent reads
329    * for the same scanner.
330    */
331   public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
332     final byte[] row = Bytes.toBytes(1);
333     final byte[] f = Bytes.toBytes("family");
334     final byte[] q1 = Bytes.toBytes("q1");
335     final byte[] q2 = Bytes.toBytes("q2");
336     final byte[] v1 = Bytes.toBytes("value1");
337     // INSERT 1: Write both columns val1
338     MultiVersionConsistencyControl.WriteEntry w =
339         mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
340 
341     KeyValue kv11 = new KeyValue(row, f, q1, v1);
342     kv11.setSequenceId(w.getWriteNumber());
343     memstore.add(kv11);
344 
345     KeyValue kv12 = new KeyValue(row, f, q2, v1);
346     kv12.setSequenceId(w.getWriteNumber());
347     memstore.add(kv12);
348     mvcc.completeMemstoreInsert(w);
349 
350     // BEFORE STARTING INSERT 2, SEE FIRST KVS
351     KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
352     assertScannerResults(s, new KeyValue[]{kv11, kv12});
353 
354     // START DELETE: Insert delete for one of the columns
355     w = mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
356     KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
357         KeyValue.Type.DeleteColumn);
358     kvDel.setSequenceId(w.getWriteNumber());
359     memstore.add(kvDel);
360 
361     // BEFORE COMPLETING DELETE, SEE FIRST KVS
362     s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
363     assertScannerResults(s, new KeyValue[]{kv11, kv12});
364 
365     // COMPLETE DELETE
366     mvcc.completeMemstoreInsert(w);
367 
368     // NOW WE SHOULD SEE DELETE
369     s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
370     assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
371   }
372 
373 
374   private static class ReadOwnWritesTester extends Thread {
375     static final int NUM_TRIES = 1000;
376 
377     final byte[] row;
378 
379     final byte[] f = Bytes.toBytes("family");
380     final byte[] q1 = Bytes.toBytes("q1");
381 
382     final MultiVersionConsistencyControl mvcc;
383     final MemStore memstore;
384     final AtomicLong startSeqNum;
385 
386     AtomicReference<Throwable> caughtException;
387 
388 
389     public ReadOwnWritesTester(int id,
390                                MemStore memstore,
391                                MultiVersionConsistencyControl mvcc,
392                                AtomicReference<Throwable> caughtException,
393                                AtomicLong startSeqNum)
394     {
395       this.mvcc = mvcc;
396       this.memstore = memstore;
397       this.caughtException = caughtException;
398       row = Bytes.toBytes(id);
399       this.startSeqNum = startSeqNum;
400     }
401 
402     public void run() {
403       try {
404         internalRun();
405       } catch (Throwable t) {
406         caughtException.compareAndSet(null, t);
407       }
408     }
409 
410     private void internalRun() throws IOException {
411       for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
412         MultiVersionConsistencyControl.WriteEntry w =
413             mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
414 
415         // Insert the sequence value (i)
416         byte[] v = Bytes.toBytes(i);
417 
418         KeyValue kv = new KeyValue(row, f, q1, i, v);
419         kv.setSequenceId(w.getWriteNumber());
420         memstore.add(kv);
421         mvcc.completeMemstoreInsert(w);
422 
423         // Assert that we can read back
424         KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
425         s.seek(kv);
426 
427         Cell ret = s.next();
428         assertNotNull("Didnt find own write at all", ret);
429         assertEquals("Didnt read own writes",
430                      kv.getTimestamp(), ret.getTimestamp());
431       }
432     }
433   }
434 
435   public void testReadOwnWritesUnderConcurrency() throws Throwable {
436 
437     int NUM_THREADS = 8;
438 
439     ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
440     AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
441 
442     for (int i = 0; i < NUM_THREADS; i++) {
443       threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught, this.startSeqNum);
444       threads[i].start();
445     }
446 
447     for (int i = 0; i < NUM_THREADS; i++) {
448       threads[i].join();
449     }
450 
451     if (caught.get() != null) {
452       throw caught.get();
453     }
454   }
455 
456   /**
457    * Test memstore snapshots
458    * @throws IOException
459    */
460   public void testSnapshotting() throws IOException {
461     final int snapshotCount = 5;
462     // Add some rows, run a snapshot. Do it a few times.
463     for (int i = 0; i < snapshotCount; i++) {
464       addRows(this.memstore);
465       runSnapshot(this.memstore);
466       assertEquals("History not being cleared", 0, this.memstore.snapshot.size());
467     }
468   }
469 
470   public void testMultipleVersionsSimple() throws Exception {
471     DefaultMemStore m = new DefaultMemStore(new Configuration(), KeyValue.COMPARATOR);
472     byte [] row = Bytes.toBytes("testRow");
473     byte [] family = Bytes.toBytes("testFamily");
474     byte [] qf = Bytes.toBytes("testQualifier");
475     long [] stamps = {1,2,3};
476     byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
477         Bytes.toBytes("value2")};
478     KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
479     KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
480     KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
481 
482     m.add(key0);
483     m.add(key1);
484     m.add(key2);
485 
486     assertTrue("Expected memstore to hold 3 values, actually has " +
487         m.cellSet.size(), m.cellSet.size() == 3);
488   }
489 
490   //////////////////////////////////////////////////////////////////////////////
491   // Get tests
492   //////////////////////////////////////////////////////////////////////////////
493 
494   /** Test getNextRow from memstore
495    * @throws InterruptedException
496    */
497   public void testGetNextRow() throws Exception {
498     addRows(this.memstore);
499     // Add more versions to make it a little more interesting.
500     Thread.sleep(1);
501     addRows(this.memstore);
502     Cell closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
503     assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
504       new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
505     for (int i = 0; i < ROW_COUNT; i++) {
506       Cell nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
507         System.currentTimeMillis()));
508       if (i + 1 == ROW_COUNT) {
509         assertEquals(nr, null);
510       } else {
511         assertTrue(KeyValue.COMPARATOR.compareRows(nr,
512           new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
513       }
514     }
515     //starting from each row, validate results should contain the starting row
516     for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
517       ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE,
518           0, this.memstore.comparator);
519       ScanType scanType = ScanType.USER_SCAN;
520       InternalScanner scanner = new StoreScanner(new Scan(
521           Bytes.toBytes(startRowId)), scanInfo, scanType, null,
522           memstore.getScanners(0));
523       List<Cell> results = new ArrayList<Cell>();
524       for (int i = 0; scanner.next(results); i++) {
525         int rowId = startRowId + i;
526         Cell left = results.get(0);
527         byte[] row1 = Bytes.toBytes(rowId);
528         assertTrue(
529             "Row name",
530             KeyValue.COMPARATOR.compareRows(left.getRowArray(), left.getRowOffset(),
531                 (int) left.getRowLength(), row1, 0, row1.length) == 0);
532         assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
533         List<Cell> row = new ArrayList<Cell>();
534         for (Cell kv : results) {
535           row.add(kv);
536         }
537         isExpectedRowWithoutTimestamps(rowId, row);
538         // Clear out set.  Otherwise row results accumulate.
539         results.clear();
540       }
541     }
542   }
543 
544   public void testGet_memstoreAndSnapShot() throws IOException {
545     byte [] row = Bytes.toBytes("testrow");
546     byte [] fam = Bytes.toBytes("testfamily");
547     byte [] qf1 = Bytes.toBytes("testqualifier1");
548     byte [] qf2 = Bytes.toBytes("testqualifier2");
549     byte [] qf3 = Bytes.toBytes("testqualifier3");
550     byte [] qf4 = Bytes.toBytes("testqualifier4");
551     byte [] qf5 = Bytes.toBytes("testqualifier5");
552     byte [] val = Bytes.toBytes("testval");
553 
554     //Setting up memstore
555     memstore.add(new KeyValue(row, fam ,qf1, val));
556     memstore.add(new KeyValue(row, fam ,qf2, val));
557     memstore.add(new KeyValue(row, fam ,qf3, val));
558     //Creating a snapshot
559     memstore.snapshot();
560     assertEquals(3, memstore.snapshot.size());
561     //Adding value to "new" memstore
562     assertEquals(0, memstore.cellSet.size());
563     memstore.add(new KeyValue(row, fam ,qf4, val));
564     memstore.add(new KeyValue(row, fam ,qf5, val));
565     assertEquals(2, memstore.cellSet.size());
566   }
567 
568   //////////////////////////////////////////////////////////////////////////////
569   // Delete tests
570   //////////////////////////////////////////////////////////////////////////////
571   public void testGetWithDelete() throws IOException {
572     byte [] row = Bytes.toBytes("testrow");
573     byte [] fam = Bytes.toBytes("testfamily");
574     byte [] qf1 = Bytes.toBytes("testqualifier");
575     byte [] val = Bytes.toBytes("testval");
576 
577     long ts1 = System.nanoTime();
578     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
579     long ts2 = ts1 + 1;
580     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
581     long ts3 = ts2 +1;
582     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
583     memstore.add(put1);
584     memstore.add(put2);
585     memstore.add(put3);
586 
587     assertEquals(3, memstore.cellSet.size());
588 
589     KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
590     memstore.delete(del2);
591 
592     List<Cell> expected = new ArrayList<Cell>();
593     expected.add(put3);
594     expected.add(del2);
595     expected.add(put2);
596     expected.add(put1);
597 
598     assertEquals(4, memstore.cellSet.size());
599     int i = 0;
600     for(Cell cell : memstore.cellSet) {
601       assertEquals(expected.get(i++), cell);
602     }
603   }
604 
605   public void testGetWithDeleteColumn() throws IOException {
606     byte [] row = Bytes.toBytes("testrow");
607     byte [] fam = Bytes.toBytes("testfamily");
608     byte [] qf1 = Bytes.toBytes("testqualifier");
609     byte [] val = Bytes.toBytes("testval");
610 
611     long ts1 = System.nanoTime();
612     KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
613     long ts2 = ts1 + 1;
614     KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
615     long ts3 = ts2 +1;
616     KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
617     memstore.add(put1);
618     memstore.add(put2);
619     memstore.add(put3);
620 
621     assertEquals(3, memstore.cellSet.size());
622 
623     KeyValue del2 =
624       new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
625     memstore.delete(del2);
626 
627     List<Cell> expected = new ArrayList<Cell>();
628     expected.add(put3);
629     expected.add(del2);
630     expected.add(put2);
631     expected.add(put1);
632 
633 
634     assertEquals(4, memstore.cellSet.size());
635     int i = 0;
636     for (Cell cell: memstore.cellSet) {
637       assertEquals(expected.get(i++), cell);
638     }
639   }
640 
641 
642   public void testGetWithDeleteFamily() throws IOException {
643     byte [] row = Bytes.toBytes("testrow");
644     byte [] fam = Bytes.toBytes("testfamily");
645     byte [] qf1 = Bytes.toBytes("testqualifier1");
646     byte [] qf2 = Bytes.toBytes("testqualifier2");
647     byte [] qf3 = Bytes.toBytes("testqualifier3");
648     byte [] val = Bytes.toBytes("testval");
649     long ts = System.nanoTime();
650 
651     KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
652     KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
653     KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
654     KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
655 
656     memstore.add(put1);
657     memstore.add(put2);
658     memstore.add(put3);
659     memstore.add(put4);
660 
661     KeyValue del =
662       new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
663     memstore.delete(del);
664 
665     List<Cell> expected = new ArrayList<Cell>();
666     expected.add(del);
667     expected.add(put1);
668     expected.add(put2);
669     expected.add(put4);
670     expected.add(put3);
671 
672 
673 
674     assertEquals(5, memstore.cellSet.size());
675     int i = 0;
676     for (Cell cell: memstore.cellSet) {
677       assertEquals(expected.get(i++), cell);
678     }
679   }
680 
681   public void testKeepDeleteInmemstore() {
682     byte [] row = Bytes.toBytes("testrow");
683     byte [] fam = Bytes.toBytes("testfamily");
684     byte [] qf = Bytes.toBytes("testqualifier");
685     byte [] val = Bytes.toBytes("testval");
686     long ts = System.nanoTime();
687     memstore.add(new KeyValue(row, fam, qf, ts, val));
688     KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
689     memstore.delete(delete);
690     assertEquals(2, memstore.cellSet.size());
691     assertEquals(delete, memstore.cellSet.first());
692   }
693 
694   public void testRetainsDeleteVersion() throws IOException {
695     // add a put to memstore
696     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
697 
698     // now process a specific delete:
699     KeyValue delete = KeyValueTestUtil.create(
700         "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
701     memstore.delete(delete);
702 
703     assertEquals(2, memstore.cellSet.size());
704     assertEquals(delete, memstore.cellSet.first());
705   }
706   public void testRetainsDeleteColumn() throws IOException {
707     // add a put to memstore
708     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
709 
710     // now process a specific delete:
711     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
712         KeyValue.Type.DeleteColumn, "dont-care");
713     memstore.delete(delete);
714 
715     assertEquals(2, memstore.cellSet.size());
716     assertEquals(delete, memstore.cellSet.first());
717   }
718   public void testRetainsDeleteFamily() throws IOException {
719     // add a put to memstore
720     memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
721 
722     // now process a specific delete:
723     KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
724         KeyValue.Type.DeleteFamily, "dont-care");
725     memstore.delete(delete);
726 
727     assertEquals(2, memstore.cellSet.size());
728     assertEquals(delete, memstore.cellSet.first());
729   }
730 
731   ////////////////////////////////////
732   //Test for timestamps
733   ////////////////////////////////////
734 
735   /**
736    * Test to ensure correctness when using Memstore with multiple timestamps
737    */
738   public void testMultipleTimestamps() throws IOException {
739     long[] timestamps = new long[] {20,10,5,1};
740     Scan scan = new Scan();
741 
742     for (long timestamp: timestamps)
743       addRows(memstore,timestamp);
744 
745     scan.setTimeRange(0, 2);
746     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
747 
748     scan.setTimeRange(20, 82);
749     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
750 
751     scan.setTimeRange(10, 20);
752     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
753 
754     scan.setTimeRange(8, 12);
755     assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
756 
757     /*This test is not required for correctness but it should pass when
758      * timestamp range optimization is on*/
759     //scan.setTimeRange(28, 42);
760     //assertTrue(!memstore.shouldSeek(scan));
761   }
762 
763   ////////////////////////////////////
764   //Test for upsert with MSLAB
765   ////////////////////////////////////
766 
767   /**
768    * Test a pathological pattern that shows why we can't currently
769    * use the MSLAB for upsert workloads. This test inserts data
770    * in the following pattern:
771    *
772    * - row0001 through row1000 (fills up one 2M Chunk)
773    * - row0002 through row1001 (fills up another 2M chunk, leaves one reference
774    *   to the first chunk
775    * - row0003 through row1002 (another chunk, another dangling reference)
776    *
777    * This causes OOME pretty quickly if we use MSLAB for upsert
778    * since each 2M chunk is held onto by a single reference.
779    */
780   public void testUpsertMSLAB() throws Exception {
781     Configuration conf = HBaseConfiguration.create();
782     conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true);
783     memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);
784 
785     int ROW_SIZE = 2048;
786     byte[] qualifier = new byte[ROW_SIZE - 4];
787 
788     MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
789     for (int i = 0; i < 3; i++) { System.gc(); }
790     long usageBefore = bean.getHeapMemoryUsage().getUsed();
791 
792     long size = 0;
793     long ts=0;
794 
795     for (int newValue = 0; newValue < 1000; newValue++) {
796       for (int row = newValue; row < newValue + 1000; row++) {
797         byte[] rowBytes = Bytes.toBytes(row);
798         size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
799       }
800     }
801     System.out.println("Wrote " + ts + " vals");
802     for (int i = 0; i < 3; i++) { System.gc(); }
803     long usageAfter = bean.getHeapMemoryUsage().getUsed();
804     System.out.println("Memory used: " + (usageAfter - usageBefore)
805         + " (heapsize: " + memstore.heapSize() +
806         " size: " + size + ")");
807   }
808 
809   //////////////////////////////////////////////////////////////////////////////
810   // Helpers
811   //////////////////////////////////////////////////////////////////////////////
812   private static byte [] makeQualifier(final int i1, final int i2){
813     return Bytes.toBytes(Integer.toString(i1) + ";" +
814         Integer.toString(i2));
815   }
816 
817   /**
818    * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
819    * as older keyvalues are deleted from the memstore.
820    * @throws Exception
821    */
822   public void testUpsertMemstoreSize() throws Exception {
823     Configuration conf = HBaseConfiguration.create();
824     memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);
825     long oldSize = memstore.size.get();
826 
827     List<Cell> l = new ArrayList<Cell>();
828     KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
829     KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
830     KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");
831 
832     kv1.setSequenceId(1); kv2.setSequenceId(1);kv3.setSequenceId(1);
833     l.add(kv1); l.add(kv2); l.add(kv3);
834 
835     this.memstore.upsert(l, 2);// readpoint is 2
836     long newSize = this.memstore.size.get();
837     assert(newSize > oldSize);
838 
839     KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
840     kv4.setSequenceId(1);
841     l.clear(); l.add(kv4);
842     this.memstore.upsert(l, 3);
843     assertEquals(newSize, this.memstore.size.get());
844     //this.memstore = null;
845   }
846 
847   ////////////////////////////////////
848   // Test for periodic memstore flushes 
849   // based on time of oldest edit
850   ////////////////////////////////////
851 
852   /**
853    * Tests that the timeOfOldestEdit is updated correctly for the 
854    * various edit operations in memstore.
855    * @throws Exception
856    */
857   public void testUpdateToTimeOfOldestEdit() throws Exception {
858     try {
859       EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
860       EnvironmentEdgeManager.injectEdge(edge);
861       DefaultMemStore memstore = new DefaultMemStore();
862       long t = memstore.timeOfOldestEdit();
863       assertEquals(t, Long.MAX_VALUE);
864 
865       // test the case that the timeOfOldestEdit is updated after a KV add
866       memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
867       t = memstore.timeOfOldestEdit();
868       assertTrue(t == 1234);
869       // snapshot() will reset timeOfOldestEdit. The method will also assert the 
870       // value is reset to Long.MAX_VALUE
871       t = runSnapshot(memstore);
872 
873       // test the case that the timeOfOldestEdit is updated after a KV delete
874       memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
875       t = memstore.timeOfOldestEdit();
876       assertTrue(t == 1234);
877       t = runSnapshot(memstore);
878 
879       // test the case that the timeOfOldestEdit is updated after a KV upsert
880       List<Cell> l = new ArrayList<Cell>();
881       KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
882       kv1.setSequenceId(100);
883       l.add(kv1);
884       memstore.upsert(l, 1000);
885       t = memstore.timeOfOldestEdit();
886       assertTrue(t == 1234);
887     } finally {
888       EnvironmentEdgeManager.reset();
889     }
890   }
891 
892   /**
893    * Tests the HRegion.shouldFlush method - adds an edit in the memstore
894    * and checks that shouldFlush returns true, and another where it disables
895    * the periodic flush functionality and tests whether shouldFlush returns
896    * false. 
897    * @throws Exception
898    */
899   public void testShouldFlush() throws Exception {
900     Configuration conf = new Configuration();
901     conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
902     checkShouldFlush(conf, true);
903     // test disable flush
904     conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
905     checkShouldFlush(conf, false);
906   }
907 
908   private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
909     try {
910       EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
911       EnvironmentEdgeManager.injectEdge(edge);
912       HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf);
913       HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
914 
915       Map<byte[], Store> stores = region.getStores();
916       assertTrue(stores.size() == 1);
917 
918       Store s = stores.entrySet().iterator().next().getValue();
919       edge.setCurrentTimeMillis(1234);
920       s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
921       edge.setCurrentTimeMillis(1234 + 100);
922       assertTrue(region.shouldFlush() == false);
923       edge.setCurrentTimeMillis(1234 + 10000);
924       assertTrue(region.shouldFlush() == expected);
925     } finally {
926       EnvironmentEdgeManager.reset();
927     }
928   }
929 
930   private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
931     long t = 1234;
932     @Override
933     public long currentTime() {
934       return t; 
935     }
936     public void setCurrentTimeMillis(long t) {
937       this.t = t;
938     }
939   }
940 
941   /**
942    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
943    * @param hmc Instance to add rows to.
944    * @return How many rows we added.
945    * @throws IOException
946    */
947   private int addRows(final MemStore hmc) {
948     return addRows(hmc, HConstants.LATEST_TIMESTAMP);
949   }
950 
951   /**
952    * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
953    * @param hmc Instance to add rows to.
954    * @return How many rows we added.
955    * @throws IOException
956    */
957   private int addRows(final MemStore hmc, final long ts) {
958     for (int i = 0; i < ROW_COUNT; i++) {
959       long timestamp = ts == HConstants.LATEST_TIMESTAMP?
960         System.currentTimeMillis(): ts;
961       for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
962         byte [] row = Bytes.toBytes(i);
963         byte [] qf = makeQualifier(i, ii);
964         hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
965       }
966     }
967     return ROW_COUNT;
968   }
969 
970   private long runSnapshot(final DefaultMemStore hmc) throws UnexpectedStateException {
971     // Save off old state.
972     int oldHistorySize = hmc.snapshot.size();
973     MemStoreSnapshot snapshot = hmc.snapshot();
974     // Make some assertions about what just happened.
975     assertTrue("History size has not increased", oldHistorySize < hmc.snapshot.size());
976     long t = memstore.timeOfOldestEdit();
977     assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
978     hmc.clearSnapshot(snapshot.getId());
979     return t;
980   }
981 
982   private void isExpectedRowWithoutTimestamps(final int rowIndex,
983       List<Cell> kvs) {
984     int i = 0;
985     for (Cell kv: kvs) {
986       byte[] expectedColname = makeQualifier(rowIndex, i++);
987       assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname));
988       // Value is column name as bytes.  Usually result is
989       // 100 bytes in size at least. This is the default size
990       // for BytesWriteable.  For comparison, convert bytes to
991       // String and trim to remove trailing null bytes.
992       assertTrue("Content", CellUtil.matchingValue(kv, expectedColname));
993     }
994   }
995 
996   private static void addRows(int count, final MemStore mem) {
997     long nanos = System.nanoTime();
998 
999     for (int i = 0 ; i < count ; i++) {
1000       if (i % 1000 == 0) {
1001 
1002         System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
1003         nanos = System.nanoTime();
1004       }
1005       long timestamp = System.currentTimeMillis();
1006 
1007       for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
1008         byte [] row = Bytes.toBytes(i);
1009         byte [] qf = makeQualifier(i, ii);
1010         mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
1011       }
1012     }
1013   }
1014 
1015 
1016   static void doScan(MemStore ms, int iteration) throws IOException {
1017     long nanos = System.nanoTime();
1018     KeyValueScanner s = ms.getScanners(0).get(0);
1019     s.seek(KeyValueUtil.createFirstOnRow(new byte[]{}));
1020 
1021     System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
1022     int cnt=0;
1023     while(s.next() != null) ++cnt;
1024 
1025     System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos) / 1000 + " for: "
1026         + cnt);
1027 
1028   }
1029 
1030   public static void main(String [] args) throws IOException {
1031     MemStore ms = new DefaultMemStore();
1032 
1033     long n1 = System.nanoTime();
1034     addRows(25000, ms);
1035     System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
1036 
1037     System.out.println("foo");
1038 
1039     for (int i = 0 ; i < 50 ; i++)
1040       doScan(ms, i);
1041   }
1042 }
1043