1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.lang.management.ManagementFactory;
23 import java.lang.management.MemoryMXBean;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.concurrent.atomic.AtomicLong;
29 import java.util.concurrent.atomic.AtomicReference;
30
31 import junit.framework.TestCase;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.Cell;
37 import org.apache.hadoop.hbase.CellUtil;
38 import org.apache.hadoop.hbase.HBaseConfiguration;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.KeepDeletedCells;
43 import org.apache.hadoop.hbase.KeyValue;
44 import org.apache.hadoop.hbase.KeyValueTestUtil;
45 import org.apache.hadoop.hbase.KeyValueUtil;
46 import org.apache.hadoop.hbase.testclassification.MediumTests;
47 import org.apache.hadoop.hbase.client.Scan;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.hbase.util.EnvironmentEdge;
50 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
51 import org.junit.experimental.categories.Category;
52
53 import com.google.common.base.Joiner;
54 import com.google.common.collect.Iterables;
55 import com.google.common.collect.Lists;
56
57
58 @Category(MediumTests.class)
59 public class TestDefaultMemStore extends TestCase {
60 private final Log LOG = LogFactory.getLog(this.getClass());
61 private DefaultMemStore memstore;
62 private static final int ROW_COUNT = 10;
63 private static final int QUALIFIER_COUNT = ROW_COUNT;
64 private static final byte [] FAMILY = Bytes.toBytes("column");
65 private MultiVersionConsistencyControl mvcc;
66 private AtomicLong startSeqNum = new AtomicLong(0);
67
68 @Override
69 public void setUp() throws Exception {
70 super.setUp();
71 this.mvcc = new MultiVersionConsistencyControl();
72 this.memstore = new DefaultMemStore();
73 }
74
75 public void testPutSameKey() {
76 byte [] bytes = Bytes.toBytes(getName());
77 KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
78 this.memstore.add(kv);
79 byte [] other = Bytes.toBytes("somethingelse");
80 KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
81 this.memstore.add(samekey);
82 Cell found = this.memstore.cellSet.first();
83 assertEquals(1, this.memstore.cellSet.size());
84 assertTrue(Bytes.toString(found.getValue()), CellUtil.matchingValue(samekey, found));
85 }
86
87
88
89
90
91 public void testScanAcrossSnapshot() throws IOException {
92 int rowCount = addRows(this.memstore);
93 List<KeyValueScanner> memstorescanners = this.memstore.getScanners(0);
94 Scan scan = new Scan();
95 List<Cell> result = new ArrayList<Cell>();
96 ScanInfo scanInfo =
97 new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0,
98 this.memstore.comparator);
99 ScanType scanType = ScanType.USER_SCAN;
100 StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
101 int count = 0;
102 try {
103 while (s.next(result)) {
104 LOG.info(result);
105 count++;
106
107 assertEquals(rowCount, result.size());
108 result.clear();
109 }
110 } finally {
111 s.close();
112 }
113 assertEquals(rowCount, count);
114 for (KeyValueScanner scanner : memstorescanners) {
115 scanner.close();
116 }
117
118 memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
119
120 s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
121 count = 0;
122 try {
123 while (s.next(result)) {
124 LOG.info(result);
125
126 assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
127 count++;
128
129 assertEquals(rowCount, result.size());
130 if (count == 2) {
131 this.memstore.snapshot();
132 LOG.info("Snapshotted");
133 }
134 result.clear();
135 }
136 } finally {
137 s.close();
138 }
139 assertEquals(rowCount, count);
140 for (KeyValueScanner scanner : memstorescanners) {
141 scanner.close();
142 }
143 memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
144
145 long ts = System.currentTimeMillis();
146 s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
147 count = 0;
148 int snapshotIndex = 5;
149 try {
150 while (s.next(result)) {
151 LOG.info(result);
152
153 assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
154
155 assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
156 count++;
157 if (count == snapshotIndex) {
158 MemStoreSnapshot snapshot = this.memstore.snapshot();
159 this.memstore.clearSnapshot(snapshot.getId());
160
161 addRows(this.memstore, ts);
162 LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
163 }
164 result.clear();
165 }
166 } finally {
167 s.close();
168 }
169 assertEquals(rowCount, count);
170 }
171
172
173
174
175
176
177 public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
178
179
180 final byte[] one = Bytes.toBytes(1);
181 final byte[] two = Bytes.toBytes(2);
182 final byte[] f = Bytes.toBytes("f");
183 final byte[] q = Bytes.toBytes("q");
184 final byte[] v = Bytes.toBytes(3);
185
186 final KeyValue kv1 = new KeyValue(one, f, q, v);
187 final KeyValue kv2 = new KeyValue(two, f, q, v);
188
189
190 this.memstore.add(kv1.clone());
191 this.memstore.add(kv2.clone());
192 verifyScanAcrossSnapshot2(kv1, kv2);
193
194
195 this.memstore.snapshot();
196 verifyScanAcrossSnapshot2(kv1, kv2);
197
198
199 this.memstore = new DefaultMemStore();
200 this.memstore.add(kv1.clone());
201 this.memstore.snapshot();
202 this.memstore.add(kv2.clone());
203 verifyScanAcrossSnapshot2(kv1, kv2);
204 }
205
206 private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
207 throws IOException {
208 List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
209 assertEquals(1, memstorescanners.size());
210 final KeyValueScanner scanner = memstorescanners.get(0);
211 scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
212 assertEquals(kv1, scanner.next());
213 assertEquals(kv2, scanner.next());
214 assertNull(scanner.next());
215 }
216
217 private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
218 throws IOException {
219 scanner.seek(KeyValueUtil.createFirstOnRow(new byte[]{}));
220 List<Cell> returned = Lists.newArrayList();
221
222 while (true) {
223 Cell next = scanner.next();
224 if (next == null) break;
225 returned.add(next);
226 }
227
228 assertTrue(
229 "Got:\n" + Joiner.on("\n").join(returned) +
230 "\nExpected:\n" + Joiner.on("\n").join(expected),
231 Iterables.elementsEqual(Arrays.asList(expected), returned));
232 assertNull(scanner.peek());
233 }
234
235 public void testMemstoreConcurrentControl() throws IOException {
236 final byte[] row = Bytes.toBytes(1);
237 final byte[] f = Bytes.toBytes("family");
238 final byte[] q1 = Bytes.toBytes("q1");
239 final byte[] q2 = Bytes.toBytes("q2");
240 final byte[] v = Bytes.toBytes("value");
241
242 MultiVersionConsistencyControl.WriteEntry w =
243 mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
244
245 KeyValue kv1 = new KeyValue(row, f, q1, v);
246 kv1.setSequenceId(w.getWriteNumber());
247 memstore.add(kv1);
248
249 KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
250 assertScannerResults(s, new KeyValue[]{});
251
252 mvcc.completeMemstoreInsert(w);
253
254 s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
255 assertScannerResults(s, new KeyValue[]{kv1});
256
257 w = mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
258 KeyValue kv2 = new KeyValue(row, f, q2, v);
259 kv2.setSequenceId(w.getWriteNumber());
260 memstore.add(kv2);
261
262 s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
263 assertScannerResults(s, new KeyValue[]{kv1});
264
265 mvcc.completeMemstoreInsert(w);
266
267 s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
268 assertScannerResults(s, new KeyValue[]{kv1, kv2});
269 }
270
271
272
273
274
275
276
277 public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
278 final byte[] row = Bytes.toBytes(1);
279 final byte[] f = Bytes.toBytes("family");
280 final byte[] q1 = Bytes.toBytes("q1");
281 final byte[] q2 = Bytes.toBytes("q2");
282 final byte[] v1 = Bytes.toBytes("value1");
283 final byte[] v2 = Bytes.toBytes("value2");
284
285
286 MultiVersionConsistencyControl.WriteEntry w =
287 mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
288
289 KeyValue kv11 = new KeyValue(row, f, q1, v1);
290 kv11.setSequenceId(w.getWriteNumber());
291 memstore.add(kv11);
292
293 KeyValue kv12 = new KeyValue(row, f, q2, v1);
294 kv12.setSequenceId(w.getWriteNumber());
295 memstore.add(kv12);
296 mvcc.completeMemstoreInsert(w);
297
298
299 KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
300 assertScannerResults(s, new KeyValue[]{kv11, kv12});
301
302
303 w = mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
304 KeyValue kv21 = new KeyValue(row, f, q1, v2);
305 kv21.setSequenceId(w.getWriteNumber());
306 memstore.add(kv21);
307
308 KeyValue kv22 = new KeyValue(row, f, q2, v2);
309 kv22.setSequenceId(w.getWriteNumber());
310 memstore.add(kv22);
311
312
313 s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
314 assertScannerResults(s, new KeyValue[]{kv11, kv12});
315
316
317 mvcc.completeMemstoreInsert(w);
318
319
320
321
322 s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
323 assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
324 }
325
326
327
328
329
330
331 public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
332 final byte[] row = Bytes.toBytes(1);
333 final byte[] f = Bytes.toBytes("family");
334 final byte[] q1 = Bytes.toBytes("q1");
335 final byte[] q2 = Bytes.toBytes("q2");
336 final byte[] v1 = Bytes.toBytes("value1");
337
338 MultiVersionConsistencyControl.WriteEntry w =
339 mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
340
341 KeyValue kv11 = new KeyValue(row, f, q1, v1);
342 kv11.setSequenceId(w.getWriteNumber());
343 memstore.add(kv11);
344
345 KeyValue kv12 = new KeyValue(row, f, q2, v1);
346 kv12.setSequenceId(w.getWriteNumber());
347 memstore.add(kv12);
348 mvcc.completeMemstoreInsert(w);
349
350
351 KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
352 assertScannerResults(s, new KeyValue[]{kv11, kv12});
353
354
355 w = mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
356 KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
357 KeyValue.Type.DeleteColumn);
358 kvDel.setSequenceId(w.getWriteNumber());
359 memstore.add(kvDel);
360
361
362 s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
363 assertScannerResults(s, new KeyValue[]{kv11, kv12});
364
365
366 mvcc.completeMemstoreInsert(w);
367
368
369 s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
370 assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
371 }
372
373
374 private static class ReadOwnWritesTester extends Thread {
375 static final int NUM_TRIES = 1000;
376
377 final byte[] row;
378
379 final byte[] f = Bytes.toBytes("family");
380 final byte[] q1 = Bytes.toBytes("q1");
381
382 final MultiVersionConsistencyControl mvcc;
383 final MemStore memstore;
384 final AtomicLong startSeqNum;
385
386 AtomicReference<Throwable> caughtException;
387
388
389 public ReadOwnWritesTester(int id,
390 MemStore memstore,
391 MultiVersionConsistencyControl mvcc,
392 AtomicReference<Throwable> caughtException,
393 AtomicLong startSeqNum)
394 {
395 this.mvcc = mvcc;
396 this.memstore = memstore;
397 this.caughtException = caughtException;
398 row = Bytes.toBytes(id);
399 this.startSeqNum = startSeqNum;
400 }
401
402 public void run() {
403 try {
404 internalRun();
405 } catch (Throwable t) {
406 caughtException.compareAndSet(null, t);
407 }
408 }
409
410 private void internalRun() throws IOException {
411 for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
412 MultiVersionConsistencyControl.WriteEntry w =
413 mvcc.beginMemstoreInsertWithSeqNum(this.startSeqNum.incrementAndGet());
414
415
416 byte[] v = Bytes.toBytes(i);
417
418 KeyValue kv = new KeyValue(row, f, q1, i, v);
419 kv.setSequenceId(w.getWriteNumber());
420 memstore.add(kv);
421 mvcc.completeMemstoreInsert(w);
422
423
424 KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
425 s.seek(kv);
426
427 Cell ret = s.next();
428 assertNotNull("Didnt find own write at all", ret);
429 assertEquals("Didnt read own writes",
430 kv.getTimestamp(), ret.getTimestamp());
431 }
432 }
433 }
434
435 public void testReadOwnWritesUnderConcurrency() throws Throwable {
436
437 int NUM_THREADS = 8;
438
439 ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
440 AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
441
442 for (int i = 0; i < NUM_THREADS; i++) {
443 threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught, this.startSeqNum);
444 threads[i].start();
445 }
446
447 for (int i = 0; i < NUM_THREADS; i++) {
448 threads[i].join();
449 }
450
451 if (caught.get() != null) {
452 throw caught.get();
453 }
454 }
455
456
457
458
459
460 public void testSnapshotting() throws IOException {
461 final int snapshotCount = 5;
462
463 for (int i = 0; i < snapshotCount; i++) {
464 addRows(this.memstore);
465 runSnapshot(this.memstore);
466 assertEquals("History not being cleared", 0, this.memstore.snapshot.size());
467 }
468 }
469
470 public void testMultipleVersionsSimple() throws Exception {
471 DefaultMemStore m = new DefaultMemStore(new Configuration(), KeyValue.COMPARATOR);
472 byte [] row = Bytes.toBytes("testRow");
473 byte [] family = Bytes.toBytes("testFamily");
474 byte [] qf = Bytes.toBytes("testQualifier");
475 long [] stamps = {1,2,3};
476 byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
477 Bytes.toBytes("value2")};
478 KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
479 KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
480 KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
481
482 m.add(key0);
483 m.add(key1);
484 m.add(key2);
485
486 assertTrue("Expected memstore to hold 3 values, actually has " +
487 m.cellSet.size(), m.cellSet.size() == 3);
488 }
489
490
491
492
493
494
495
496
497 public void testGetNextRow() throws Exception {
498 addRows(this.memstore);
499
500 Thread.sleep(1);
501 addRows(this.memstore);
502 Cell closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
503 assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
504 new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
505 for (int i = 0; i < ROW_COUNT; i++) {
506 Cell nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
507 System.currentTimeMillis()));
508 if (i + 1 == ROW_COUNT) {
509 assertEquals(nr, null);
510 } else {
511 assertTrue(KeyValue.COMPARATOR.compareRows(nr,
512 new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
513 }
514 }
515
516 for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
517 ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE,
518 0, this.memstore.comparator);
519 ScanType scanType = ScanType.USER_SCAN;
520 InternalScanner scanner = new StoreScanner(new Scan(
521 Bytes.toBytes(startRowId)), scanInfo, scanType, null,
522 memstore.getScanners(0));
523 List<Cell> results = new ArrayList<Cell>();
524 for (int i = 0; scanner.next(results); i++) {
525 int rowId = startRowId + i;
526 Cell left = results.get(0);
527 byte[] row1 = Bytes.toBytes(rowId);
528 assertTrue(
529 "Row name",
530 KeyValue.COMPARATOR.compareRows(left.getRowArray(), left.getRowOffset(),
531 (int) left.getRowLength(), row1, 0, row1.length) == 0);
532 assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
533 List<Cell> row = new ArrayList<Cell>();
534 for (Cell kv : results) {
535 row.add(kv);
536 }
537 isExpectedRowWithoutTimestamps(rowId, row);
538
539 results.clear();
540 }
541 }
542 }
543
544 public void testGet_memstoreAndSnapShot() throws IOException {
545 byte [] row = Bytes.toBytes("testrow");
546 byte [] fam = Bytes.toBytes("testfamily");
547 byte [] qf1 = Bytes.toBytes("testqualifier1");
548 byte [] qf2 = Bytes.toBytes("testqualifier2");
549 byte [] qf3 = Bytes.toBytes("testqualifier3");
550 byte [] qf4 = Bytes.toBytes("testqualifier4");
551 byte [] qf5 = Bytes.toBytes("testqualifier5");
552 byte [] val = Bytes.toBytes("testval");
553
554
555 memstore.add(new KeyValue(row, fam ,qf1, val));
556 memstore.add(new KeyValue(row, fam ,qf2, val));
557 memstore.add(new KeyValue(row, fam ,qf3, val));
558
559 memstore.snapshot();
560 assertEquals(3, memstore.snapshot.size());
561
562 assertEquals(0, memstore.cellSet.size());
563 memstore.add(new KeyValue(row, fam ,qf4, val));
564 memstore.add(new KeyValue(row, fam ,qf5, val));
565 assertEquals(2, memstore.cellSet.size());
566 }
567
568
569
570
571 public void testGetWithDelete() throws IOException {
572 byte [] row = Bytes.toBytes("testrow");
573 byte [] fam = Bytes.toBytes("testfamily");
574 byte [] qf1 = Bytes.toBytes("testqualifier");
575 byte [] val = Bytes.toBytes("testval");
576
577 long ts1 = System.nanoTime();
578 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
579 long ts2 = ts1 + 1;
580 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
581 long ts3 = ts2 +1;
582 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
583 memstore.add(put1);
584 memstore.add(put2);
585 memstore.add(put3);
586
587 assertEquals(3, memstore.cellSet.size());
588
589 KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
590 memstore.delete(del2);
591
592 List<Cell> expected = new ArrayList<Cell>();
593 expected.add(put3);
594 expected.add(del2);
595 expected.add(put2);
596 expected.add(put1);
597
598 assertEquals(4, memstore.cellSet.size());
599 int i = 0;
600 for(Cell cell : memstore.cellSet) {
601 assertEquals(expected.get(i++), cell);
602 }
603 }
604
605 public void testGetWithDeleteColumn() throws IOException {
606 byte [] row = Bytes.toBytes("testrow");
607 byte [] fam = Bytes.toBytes("testfamily");
608 byte [] qf1 = Bytes.toBytes("testqualifier");
609 byte [] val = Bytes.toBytes("testval");
610
611 long ts1 = System.nanoTime();
612 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
613 long ts2 = ts1 + 1;
614 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
615 long ts3 = ts2 +1;
616 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
617 memstore.add(put1);
618 memstore.add(put2);
619 memstore.add(put3);
620
621 assertEquals(3, memstore.cellSet.size());
622
623 KeyValue del2 =
624 new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
625 memstore.delete(del2);
626
627 List<Cell> expected = new ArrayList<Cell>();
628 expected.add(put3);
629 expected.add(del2);
630 expected.add(put2);
631 expected.add(put1);
632
633
634 assertEquals(4, memstore.cellSet.size());
635 int i = 0;
636 for (Cell cell: memstore.cellSet) {
637 assertEquals(expected.get(i++), cell);
638 }
639 }
640
641
642 public void testGetWithDeleteFamily() throws IOException {
643 byte [] row = Bytes.toBytes("testrow");
644 byte [] fam = Bytes.toBytes("testfamily");
645 byte [] qf1 = Bytes.toBytes("testqualifier1");
646 byte [] qf2 = Bytes.toBytes("testqualifier2");
647 byte [] qf3 = Bytes.toBytes("testqualifier3");
648 byte [] val = Bytes.toBytes("testval");
649 long ts = System.nanoTime();
650
651 KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
652 KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
653 KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
654 KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
655
656 memstore.add(put1);
657 memstore.add(put2);
658 memstore.add(put3);
659 memstore.add(put4);
660
661 KeyValue del =
662 new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
663 memstore.delete(del);
664
665 List<Cell> expected = new ArrayList<Cell>();
666 expected.add(del);
667 expected.add(put1);
668 expected.add(put2);
669 expected.add(put4);
670 expected.add(put3);
671
672
673
674 assertEquals(5, memstore.cellSet.size());
675 int i = 0;
676 for (Cell cell: memstore.cellSet) {
677 assertEquals(expected.get(i++), cell);
678 }
679 }
680
681 public void testKeepDeleteInmemstore() {
682 byte [] row = Bytes.toBytes("testrow");
683 byte [] fam = Bytes.toBytes("testfamily");
684 byte [] qf = Bytes.toBytes("testqualifier");
685 byte [] val = Bytes.toBytes("testval");
686 long ts = System.nanoTime();
687 memstore.add(new KeyValue(row, fam, qf, ts, val));
688 KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
689 memstore.delete(delete);
690 assertEquals(2, memstore.cellSet.size());
691 assertEquals(delete, memstore.cellSet.first());
692 }
693
694 public void testRetainsDeleteVersion() throws IOException {
695
696 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
697
698
699 KeyValue delete = KeyValueTestUtil.create(
700 "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
701 memstore.delete(delete);
702
703 assertEquals(2, memstore.cellSet.size());
704 assertEquals(delete, memstore.cellSet.first());
705 }
706 public void testRetainsDeleteColumn() throws IOException {
707
708 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
709
710
711 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
712 KeyValue.Type.DeleteColumn, "dont-care");
713 memstore.delete(delete);
714
715 assertEquals(2, memstore.cellSet.size());
716 assertEquals(delete, memstore.cellSet.first());
717 }
718 public void testRetainsDeleteFamily() throws IOException {
719
720 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
721
722
723 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
724 KeyValue.Type.DeleteFamily, "dont-care");
725 memstore.delete(delete);
726
727 assertEquals(2, memstore.cellSet.size());
728 assertEquals(delete, memstore.cellSet.first());
729 }
730
731
732
733
734
735
736
737
738 public void testMultipleTimestamps() throws IOException {
739 long[] timestamps = new long[] {20,10,5,1};
740 Scan scan = new Scan();
741
742 for (long timestamp: timestamps)
743 addRows(memstore,timestamp);
744
745 scan.setTimeRange(0, 2);
746 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
747
748 scan.setTimeRange(20, 82);
749 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
750
751 scan.setTimeRange(10, 20);
752 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
753
754 scan.setTimeRange(8, 12);
755 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
756
757
758
759
760
761 }
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780 public void testUpsertMSLAB() throws Exception {
781 Configuration conf = HBaseConfiguration.create();
782 conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true);
783 memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);
784
785 int ROW_SIZE = 2048;
786 byte[] qualifier = new byte[ROW_SIZE - 4];
787
788 MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
789 for (int i = 0; i < 3; i++) { System.gc(); }
790 long usageBefore = bean.getHeapMemoryUsage().getUsed();
791
792 long size = 0;
793 long ts=0;
794
795 for (int newValue = 0; newValue < 1000; newValue++) {
796 for (int row = newValue; row < newValue + 1000; row++) {
797 byte[] rowBytes = Bytes.toBytes(row);
798 size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
799 }
800 }
801 System.out.println("Wrote " + ts + " vals");
802 for (int i = 0; i < 3; i++) { System.gc(); }
803 long usageAfter = bean.getHeapMemoryUsage().getUsed();
804 System.out.println("Memory used: " + (usageAfter - usageBefore)
805 + " (heapsize: " + memstore.heapSize() +
806 " size: " + size + ")");
807 }
808
809
810
811
812 private static byte [] makeQualifier(final int i1, final int i2){
813 return Bytes.toBytes(Integer.toString(i1) + ";" +
814 Integer.toString(i2));
815 }
816
817
818
819
820
821
822 public void testUpsertMemstoreSize() throws Exception {
823 Configuration conf = HBaseConfiguration.create();
824 memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);
825 long oldSize = memstore.size.get();
826
827 List<Cell> l = new ArrayList<Cell>();
828 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
829 KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
830 KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");
831
832 kv1.setSequenceId(1); kv2.setSequenceId(1);kv3.setSequenceId(1);
833 l.add(kv1); l.add(kv2); l.add(kv3);
834
835 this.memstore.upsert(l, 2);
836 long newSize = this.memstore.size.get();
837 assert(newSize > oldSize);
838
839 assert(memstore.cellSet.size() == 2);
840
841 KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
842 kv4.setSequenceId(1);
843 l.clear(); l.add(kv4);
844 this.memstore.upsert(l, 3);
845 assertEquals(newSize, this.memstore.size.get());
846
847 assert(memstore.cellSet.size() == 2);
848
849 }
850
851
852
853
854
855
856
857
858
859
860
861 public void testUpdateToTimeOfOldestEdit() throws Exception {
862 try {
863 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
864 EnvironmentEdgeManager.injectEdge(edge);
865 DefaultMemStore memstore = new DefaultMemStore();
866 long t = memstore.timeOfOldestEdit();
867 assertEquals(t, Long.MAX_VALUE);
868
869
870 memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
871 t = memstore.timeOfOldestEdit();
872 assertTrue(t == 1234);
873
874
875 t = runSnapshot(memstore);
876
877
878 memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
879 t = memstore.timeOfOldestEdit();
880 assertTrue(t == 1234);
881 t = runSnapshot(memstore);
882
883
884 List<Cell> l = new ArrayList<Cell>();
885 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
886 kv1.setSequenceId(100);
887 l.add(kv1);
888 memstore.upsert(l, 1000);
889 t = memstore.timeOfOldestEdit();
890 assertTrue(t == 1234);
891 } finally {
892 EnvironmentEdgeManager.reset();
893 }
894 }
895
896
897
898
899
900
901
902
903 public void testShouldFlush() throws Exception {
904 Configuration conf = new Configuration();
905 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
906 checkShouldFlush(conf, true);
907
908 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
909 checkShouldFlush(conf, false);
910 }
911
912 private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
913 try {
914 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
915 EnvironmentEdgeManager.injectEdge(edge);
916 HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf);
917 HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
918
919 Map<byte[], Store> stores = region.getStores();
920 assertTrue(stores.size() == 1);
921
922 Store s = stores.entrySet().iterator().next().getValue();
923 edge.setCurrentTimeMillis(1234);
924 s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
925 edge.setCurrentTimeMillis(1234 + 100);
926 assertTrue(region.shouldFlush() == false);
927 edge.setCurrentTimeMillis(1234 + 10000);
928 assertTrue(region.shouldFlush() == expected);
929 } finally {
930 EnvironmentEdgeManager.reset();
931 }
932 }
933
934 private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
935 long t = 1234;
936 @Override
937 public long currentTime() {
938 return t;
939 }
940 public void setCurrentTimeMillis(long t) {
941 this.t = t;
942 }
943 }
944
945
946
947
948
949
950
951 private int addRows(final MemStore hmc) {
952 return addRows(hmc, HConstants.LATEST_TIMESTAMP);
953 }
954
955
956
957
958
959
960
961 private int addRows(final MemStore hmc, final long ts) {
962 for (int i = 0; i < ROW_COUNT; i++) {
963 long timestamp = ts == HConstants.LATEST_TIMESTAMP?
964 System.currentTimeMillis(): ts;
965 for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
966 byte [] row = Bytes.toBytes(i);
967 byte [] qf = makeQualifier(i, ii);
968 hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
969 }
970 }
971 return ROW_COUNT;
972 }
973
974 private long runSnapshot(final DefaultMemStore hmc) throws UnexpectedStateException {
975
976 int oldHistorySize = hmc.snapshot.size();
977 MemStoreSnapshot snapshot = hmc.snapshot();
978
979 assertTrue("History size has not increased", oldHistorySize < hmc.snapshot.size());
980 long t = memstore.timeOfOldestEdit();
981 assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
982 hmc.clearSnapshot(snapshot.getId());
983 return t;
984 }
985
986 private void isExpectedRowWithoutTimestamps(final int rowIndex,
987 List<Cell> kvs) {
988 int i = 0;
989 for (Cell kv: kvs) {
990 byte[] expectedColname = makeQualifier(rowIndex, i++);
991 assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname));
992
993
994
995
996 assertTrue("Content", CellUtil.matchingValue(kv, expectedColname));
997 }
998 }
999
1000 private static void addRows(int count, final MemStore mem) {
1001 long nanos = System.nanoTime();
1002
1003 for (int i = 0 ; i < count ; i++) {
1004 if (i % 1000 == 0) {
1005
1006 System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
1007 nanos = System.nanoTime();
1008 }
1009 long timestamp = System.currentTimeMillis();
1010
1011 for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
1012 byte [] row = Bytes.toBytes(i);
1013 byte [] qf = makeQualifier(i, ii);
1014 mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
1015 }
1016 }
1017 }
1018
1019
1020 static void doScan(MemStore ms, int iteration) throws IOException {
1021 long nanos = System.nanoTime();
1022 KeyValueScanner s = ms.getScanners(0).get(0);
1023 s.seek(KeyValueUtil.createFirstOnRow(new byte[]{}));
1024
1025 System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
1026 int cnt=0;
1027 while(s.next() != null) ++cnt;
1028
1029 System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos) / 1000 + " for: "
1030 + cnt);
1031
1032 }
1033
1034 public static void main(String [] args) throws IOException {
1035 MemStore ms = new DefaultMemStore();
1036
1037 long n1 = System.nanoTime();
1038 addRows(25000, ms);
1039 System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
1040
1041 System.out.println("foo");
1042
1043 for (int i = 0 ; i < 50 ; i++)
1044 doScan(ms, i);
1045 }
1046 }
1047