1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.lang.management.ManagementFactory;
23 import java.lang.management.MemoryMXBean;
24 import java.rmi.UnexpectedException;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.concurrent.atomic.AtomicReference;
30
31 import junit.framework.TestCase;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.Cell;
37 import org.apache.hadoop.hbase.CellUtil;
38 import org.apache.hadoop.hbase.HBaseConfiguration;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.KeyValue;
43 import org.apache.hadoop.hbase.KeyValueTestUtil;
44 import org.apache.hadoop.hbase.MediumTests;
45 import org.apache.hadoop.hbase.client.Scan;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.EnvironmentEdge;
48 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
49 import org.junit.experimental.categories.Category;
50
51 import com.google.common.base.Joiner;
52 import com.google.common.collect.Iterables;
53 import com.google.common.collect.Lists;
54
55
56 @Category(MediumTests.class)
57 public class TestMemStore extends TestCase {
58 private final Log LOG = LogFactory.getLog(this.getClass());
59 private MemStore memstore;
60 private static final int ROW_COUNT = 10;
61 private static final int QUALIFIER_COUNT = ROW_COUNT;
62 private static final byte [] FAMILY = Bytes.toBytes("column");
63 private static final byte [] CONTENTS = Bytes.toBytes("contents");
64 private static final byte [] BASIC = Bytes.toBytes("basic");
65 private static final String CONTENTSTR = "contentstr";
66 private MultiVersionConsistencyControl mvcc;
67
68 @Override
69 public void setUp() throws Exception {
70 super.setUp();
71 this.mvcc = new MultiVersionConsistencyControl();
72 this.memstore = new MemStore();
73 }
74
75 public void testPutSameKey() {
76 byte [] bytes = Bytes.toBytes(getName());
77 KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
78 this.memstore.add(kv);
79 byte [] other = Bytes.toBytes("somethingelse");
80 KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
81 this.memstore.add(samekey);
82 KeyValue found = this.memstore.kvset.first();
83 assertEquals(1, this.memstore.kvset.size());
84 assertTrue(Bytes.toString(found.getValue()), CellUtil.matchingValue(samekey, found));
85 }
86
87
88
89
90
91 public void testScanAcrossSnapshot() throws IOException {
92 int rowCount = addRows(this.memstore);
93 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
94 Scan scan = new Scan();
95 List<Cell> result = new ArrayList<Cell>();
96 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
97 ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false,
98 0, this.memstore.comparator);
99 ScanType scanType = ScanType.USER_SCAN;
100 StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
101 int count = 0;
102 try {
103 while (s.next(result)) {
104 LOG.info(result);
105 count++;
106
107 assertEquals(rowCount, result.size());
108 result.clear();
109 }
110 } finally {
111 s.close();
112 }
113 assertEquals(rowCount, count);
114 for (KeyValueScanner scanner : memstorescanners) {
115 scanner.close();
116 }
117
118 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
119 memstorescanners = this.memstore.getScanners();
120
121 s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
122 count = 0;
123 try {
124 while (s.next(result)) {
125 LOG.info(result);
126
127 assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
128 count++;
129
130 assertEquals(rowCount, result.size());
131 if (count == 2) {
132 this.memstore.snapshot();
133 LOG.info("Snapshotted");
134 }
135 result.clear();
136 }
137 } finally {
138 s.close();
139 }
140 assertEquals(rowCount, count);
141 for (KeyValueScanner scanner : memstorescanners) {
142 scanner.close();
143 }
144 memstorescanners = this.memstore.getScanners();
145
146 long ts = System.currentTimeMillis();
147 s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
148 count = 0;
149 int snapshotIndex = 5;
150 try {
151 while (s.next(result)) {
152 LOG.info(result);
153
154 assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
155
156 assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
157 count++;
158 if (count == snapshotIndex) {
159 this.memstore.snapshot();
160 this.memstore.clearSnapshot(this.memstore.getSnapshot());
161
162 addRows(this.memstore, ts);
163 LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
164 }
165 result.clear();
166 }
167 } finally {
168 s.close();
169 }
170 assertEquals(rowCount, count);
171 }
172
173
174
175
176
177
178 public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
179
180
181 final byte[] one = Bytes.toBytes(1);
182 final byte[] two = Bytes.toBytes(2);
183 final byte[] f = Bytes.toBytes("f");
184 final byte[] q = Bytes.toBytes("q");
185 final byte[] v = Bytes.toBytes(3);
186
187 final KeyValue kv1 = new KeyValue(one, f, q, v);
188 final KeyValue kv2 = new KeyValue(two, f, q, v);
189
190
191 this.memstore.add(kv1.clone());
192 this.memstore.add(kv2.clone());
193 verifyScanAcrossSnapshot2(kv1, kv2);
194
195
196 this.memstore.snapshot();
197 verifyScanAcrossSnapshot2(kv1, kv2);
198
199
200 this.memstore = new MemStore();
201 this.memstore.add(kv1.clone());
202 this.memstore.snapshot();
203 this.memstore.add(kv2.clone());
204 verifyScanAcrossSnapshot2(kv1, kv2);
205 }
206
207 private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
208 throws IOException {
209 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
210 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
211 assertEquals(1, memstorescanners.size());
212 final KeyValueScanner scanner = memstorescanners.get(0);
213 scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
214 assertEquals(kv1, scanner.next());
215 assertEquals(kv2, scanner.next());
216 assertNull(scanner.next());
217 }
218
219 private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
220 throws IOException {
221 scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
222 List<Cell> returned = Lists.newArrayList();
223
224 while (true) {
225 KeyValue next = scanner.next();
226 if (next == null) break;
227 returned.add(next);
228 }
229
230 assertTrue(
231 "Got:\n" + Joiner.on("\n").join(returned) +
232 "\nExpected:\n" + Joiner.on("\n").join(expected),
233 Iterables.elementsEqual(Arrays.asList(expected), returned));
234 assertNull(scanner.peek());
235 }
236
237 public void testMemstoreConcurrentControl() throws IOException {
238 final byte[] row = Bytes.toBytes(1);
239 final byte[] f = Bytes.toBytes("family");
240 final byte[] q1 = Bytes.toBytes("q1");
241 final byte[] q2 = Bytes.toBytes("q2");
242 final byte[] v = Bytes.toBytes("value");
243
244 MultiVersionConsistencyControl.WriteEntry w =
245 mvcc.beginMemstoreInsert();
246
247 KeyValue kv1 = new KeyValue(row, f, q1, v);
248 kv1.setMvccVersion(w.getWriteNumber());
249 memstore.add(kv1);
250
251 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
252 KeyValueScanner s = this.memstore.getScanners().get(0);
253 assertScannerResults(s, new KeyValue[]{});
254
255 mvcc.completeMemstoreInsert(w);
256
257 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
258 s = this.memstore.getScanners().get(0);
259 assertScannerResults(s, new KeyValue[]{kv1});
260
261 w = mvcc.beginMemstoreInsert();
262 KeyValue kv2 = new KeyValue(row, f, q2, v);
263 kv2.setMvccVersion(w.getWriteNumber());
264 memstore.add(kv2);
265
266 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
267 s = this.memstore.getScanners().get(0);
268 assertScannerResults(s, new KeyValue[]{kv1});
269
270 mvcc.completeMemstoreInsert(w);
271
272 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
273 s = this.memstore.getScanners().get(0);
274 assertScannerResults(s, new KeyValue[]{kv1, kv2});
275 }
276
277
278
279
280
281
282
283 public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
284 final byte[] row = Bytes.toBytes(1);
285 final byte[] f = Bytes.toBytes("family");
286 final byte[] q1 = Bytes.toBytes("q1");
287 final byte[] q2 = Bytes.toBytes("q2");
288 final byte[] v1 = Bytes.toBytes("value1");
289 final byte[] v2 = Bytes.toBytes("value2");
290
291
292 MultiVersionConsistencyControl.WriteEntry w =
293 mvcc.beginMemstoreInsert();
294
295 KeyValue kv11 = new KeyValue(row, f, q1, v1);
296 kv11.setMvccVersion(w.getWriteNumber());
297 memstore.add(kv11);
298
299 KeyValue kv12 = new KeyValue(row, f, q2, v1);
300 kv12.setMvccVersion(w.getWriteNumber());
301 memstore.add(kv12);
302 mvcc.completeMemstoreInsert(w);
303
304
305 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
306 KeyValueScanner s = this.memstore.getScanners().get(0);
307 assertScannerResults(s, new KeyValue[]{kv11, kv12});
308
309
310 w = mvcc.beginMemstoreInsert();
311 KeyValue kv21 = new KeyValue(row, f, q1, v2);
312 kv21.setMvccVersion(w.getWriteNumber());
313 memstore.add(kv21);
314
315 KeyValue kv22 = new KeyValue(row, f, q2, v2);
316 kv22.setMvccVersion(w.getWriteNumber());
317 memstore.add(kv22);
318
319
320 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
321 s = this.memstore.getScanners().get(0);
322 assertScannerResults(s, new KeyValue[]{kv11, kv12});
323
324
325 mvcc.completeMemstoreInsert(w);
326
327
328
329
330 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
331 s = this.memstore.getScanners().get(0);
332 assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
333 }
334
335
336
337
338
339
340 public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
341 final byte[] row = Bytes.toBytes(1);
342 final byte[] f = Bytes.toBytes("family");
343 final byte[] q1 = Bytes.toBytes("q1");
344 final byte[] q2 = Bytes.toBytes("q2");
345 final byte[] v1 = Bytes.toBytes("value1");
346
347 MultiVersionConsistencyControl.WriteEntry w =
348 mvcc.beginMemstoreInsert();
349
350 KeyValue kv11 = new KeyValue(row, f, q1, v1);
351 kv11.setMvccVersion(w.getWriteNumber());
352 memstore.add(kv11);
353
354 KeyValue kv12 = new KeyValue(row, f, q2, v1);
355 kv12.setMvccVersion(w.getWriteNumber());
356 memstore.add(kv12);
357 mvcc.completeMemstoreInsert(w);
358
359
360 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
361 KeyValueScanner s = this.memstore.getScanners().get(0);
362 assertScannerResults(s, new KeyValue[]{kv11, kv12});
363
364
365 w = mvcc.beginMemstoreInsert();
366 KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
367 KeyValue.Type.DeleteColumn);
368 kvDel.setMvccVersion(w.getWriteNumber());
369 memstore.add(kvDel);
370
371
372 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
373 s = this.memstore.getScanners().get(0);
374 assertScannerResults(s, new KeyValue[]{kv11, kv12});
375
376
377 mvcc.completeMemstoreInsert(w);
378
379
380 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
381 s = this.memstore.getScanners().get(0);
382 assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
383 }
384
385
386 private static class ReadOwnWritesTester extends Thread {
387 static final int NUM_TRIES = 1000;
388
389 final byte[] row;
390
391 final byte[] f = Bytes.toBytes("family");
392 final byte[] q1 = Bytes.toBytes("q1");
393
394 final MultiVersionConsistencyControl mvcc;
395 final MemStore memstore;
396
397 AtomicReference<Throwable> caughtException;
398
399
400 public ReadOwnWritesTester(int id,
401 MemStore memstore,
402 MultiVersionConsistencyControl mvcc,
403 AtomicReference<Throwable> caughtException)
404 {
405 this.mvcc = mvcc;
406 this.memstore = memstore;
407 this.caughtException = caughtException;
408 row = Bytes.toBytes(id);
409 }
410
411 public void run() {
412 try {
413 internalRun();
414 } catch (Throwable t) {
415 caughtException.compareAndSet(null, t);
416 }
417 }
418
419 private void internalRun() throws IOException {
420 for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
421 MultiVersionConsistencyControl.WriteEntry w =
422 mvcc.beginMemstoreInsert();
423
424
425 byte[] v = Bytes.toBytes(i);
426
427 KeyValue kv = new KeyValue(row, f, q1, i, v);
428 kv.setMvccVersion(w.getWriteNumber());
429 memstore.add(kv);
430 mvcc.completeMemstoreInsert(w);
431
432
433 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
434
435 KeyValueScanner s = this.memstore.getScanners().get(0);
436 s.seek(kv);
437
438 KeyValue ret = s.next();
439 assertNotNull("Didnt find own write at all", ret);
440 assertEquals("Didnt read own writes",
441 kv.getTimestamp(), ret.getTimestamp());
442 }
443 }
444 }
445
446 public void testReadOwnWritesUnderConcurrency() throws Throwable {
447
448 int NUM_THREADS = 8;
449
450 ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
451 AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
452
453 for (int i = 0; i < NUM_THREADS; i++) {
454 threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
455 threads[i].start();
456 }
457
458 for (int i = 0; i < NUM_THREADS; i++) {
459 threads[i].join();
460 }
461
462 if (caught.get() != null) {
463 throw caught.get();
464 }
465 }
466
467
468
469
470
471 public void testSnapshotting() throws IOException {
472 final int snapshotCount = 5;
473
474 for (int i = 0; i < snapshotCount; i++) {
475 addRows(this.memstore);
476 runSnapshot(this.memstore);
477 KeyValueSkipListSet ss = this.memstore.getSnapshot();
478 assertEquals("History not being cleared", 0, ss.size());
479 }
480 }
481
482 public void testMultipleVersionsSimple() throws Exception {
483 MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
484 byte [] row = Bytes.toBytes("testRow");
485 byte [] family = Bytes.toBytes("testFamily");
486 byte [] qf = Bytes.toBytes("testQualifier");
487 long [] stamps = {1,2,3};
488 byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
489 Bytes.toBytes("value2")};
490 KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
491 KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
492 KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
493
494 m.add(key0);
495 m.add(key1);
496 m.add(key2);
497
498 assertTrue("Expected memstore to hold 3 values, actually has " +
499 m.kvset.size(), m.kvset.size() == 3);
500 }
501
502
503
504
505
506
507
508
509 public void testGetNextRow() throws Exception {
510 MultiVersionConsistencyControl.resetThreadReadPoint();
511 addRows(this.memstore);
512
513 Thread.sleep(1);
514 addRows(this.memstore);
515 KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
516 assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
517 new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
518 for (int i = 0; i < ROW_COUNT; i++) {
519 KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
520 System.currentTimeMillis()));
521 if (i + 1 == ROW_COUNT) {
522 assertEquals(nr, null);
523 } else {
524 assertTrue(KeyValue.COMPARATOR.compareRows(nr,
525 new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
526 }
527 }
528
529 for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
530 ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false,
531 0, this.memstore.comparator);
532 ScanType scanType = ScanType.USER_SCAN;
533 InternalScanner scanner = new StoreScanner(new Scan(
534 Bytes.toBytes(startRowId)), scanInfo, scanType, null,
535 memstore.getScanners());
536 List<Cell> results = new ArrayList<Cell>();
537 for (int i = 0; scanner.next(results); i++) {
538 int rowId = startRowId + i;
539 Cell left = results.get(0);
540 byte[] row1 = Bytes.toBytes(rowId);
541 assertTrue("Row name",
542 KeyValue.COMPARATOR.compareRows(left.getRowArray(), left.getRowOffset(), (int) left.getRowLength(), row1, 0, row1.length) == 0);
543 assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
544 List<Cell> row = new ArrayList<Cell>();
545 for (Cell kv : results) {
546 row.add(kv);
547 }
548 isExpectedRowWithoutTimestamps(rowId, row);
549
550 results.clear();
551 }
552 }
553 }
554
555 public void testGet_memstoreAndSnapShot() throws IOException {
556 byte [] row = Bytes.toBytes("testrow");
557 byte [] fam = Bytes.toBytes("testfamily");
558 byte [] qf1 = Bytes.toBytes("testqualifier1");
559 byte [] qf2 = Bytes.toBytes("testqualifier2");
560 byte [] qf3 = Bytes.toBytes("testqualifier3");
561 byte [] qf4 = Bytes.toBytes("testqualifier4");
562 byte [] qf5 = Bytes.toBytes("testqualifier5");
563 byte [] val = Bytes.toBytes("testval");
564
565
566 memstore.add(new KeyValue(row, fam ,qf1, val));
567 memstore.add(new KeyValue(row, fam ,qf2, val));
568 memstore.add(new KeyValue(row, fam ,qf3, val));
569
570 memstore.snapshot();
571 assertEquals(3, memstore.snapshot.size());
572
573 assertEquals(0, memstore.kvset.size());
574 memstore.add(new KeyValue(row, fam ,qf4, val));
575 memstore.add(new KeyValue(row, fam ,qf5, val));
576 assertEquals(2, memstore.kvset.size());
577 }
578
579
580
581
582 public void testGetWithDelete() throws IOException {
583 byte [] row = Bytes.toBytes("testrow");
584 byte [] fam = Bytes.toBytes("testfamily");
585 byte [] qf1 = Bytes.toBytes("testqualifier");
586 byte [] val = Bytes.toBytes("testval");
587
588 long ts1 = System.nanoTime();
589 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
590 long ts2 = ts1 + 1;
591 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
592 long ts3 = ts2 +1;
593 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
594 memstore.add(put1);
595 memstore.add(put2);
596 memstore.add(put3);
597
598 assertEquals(3, memstore.kvset.size());
599
600 KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
601 memstore.delete(del2);
602
603 List<Cell> expected = new ArrayList<Cell>();
604 expected.add(put3);
605 expected.add(del2);
606 expected.add(put2);
607 expected.add(put1);
608
609 assertEquals(4, memstore.kvset.size());
610 int i = 0;
611 for(KeyValue kv : memstore.kvset) {
612 assertEquals(expected.get(i++), kv);
613 }
614 }
615
616 public void testGetWithDeleteColumn() throws IOException {
617 byte [] row = Bytes.toBytes("testrow");
618 byte [] fam = Bytes.toBytes("testfamily");
619 byte [] qf1 = Bytes.toBytes("testqualifier");
620 byte [] val = Bytes.toBytes("testval");
621
622 long ts1 = System.nanoTime();
623 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
624 long ts2 = ts1 + 1;
625 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
626 long ts3 = ts2 +1;
627 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
628 memstore.add(put1);
629 memstore.add(put2);
630 memstore.add(put3);
631
632 assertEquals(3, memstore.kvset.size());
633
634 KeyValue del2 =
635 new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
636 memstore.delete(del2);
637
638 List<Cell> expected = new ArrayList<Cell>();
639 expected.add(put3);
640 expected.add(del2);
641 expected.add(put2);
642 expected.add(put1);
643
644
645 assertEquals(4, memstore.kvset.size());
646 int i = 0;
647 for (KeyValue kv: memstore.kvset) {
648 assertEquals(expected.get(i++), kv);
649 }
650 }
651
652
653 public void testGetWithDeleteFamily() throws IOException {
654 byte [] row = Bytes.toBytes("testrow");
655 byte [] fam = Bytes.toBytes("testfamily");
656 byte [] qf1 = Bytes.toBytes("testqualifier1");
657 byte [] qf2 = Bytes.toBytes("testqualifier2");
658 byte [] qf3 = Bytes.toBytes("testqualifier3");
659 byte [] val = Bytes.toBytes("testval");
660 long ts = System.nanoTime();
661
662 KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
663 KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
664 KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
665 KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
666
667 memstore.add(put1);
668 memstore.add(put2);
669 memstore.add(put3);
670 memstore.add(put4);
671
672 KeyValue del =
673 new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
674 memstore.delete(del);
675
676 List<Cell> expected = new ArrayList<Cell>();
677 expected.add(del);
678 expected.add(put1);
679 expected.add(put2);
680 expected.add(put4);
681 expected.add(put3);
682
683
684
685 assertEquals(5, memstore.kvset.size());
686 int i = 0;
687 for (KeyValue kv: memstore.kvset) {
688 assertEquals(expected.get(i++), kv);
689 }
690 }
691
692 public void testKeepDeleteInmemstore() {
693 byte [] row = Bytes.toBytes("testrow");
694 byte [] fam = Bytes.toBytes("testfamily");
695 byte [] qf = Bytes.toBytes("testqualifier");
696 byte [] val = Bytes.toBytes("testval");
697 long ts = System.nanoTime();
698 memstore.add(new KeyValue(row, fam, qf, ts, val));
699 KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
700 memstore.delete(delete);
701 assertEquals(2, memstore.kvset.size());
702 assertEquals(delete, memstore.kvset.first());
703 }
704
705 public void testRetainsDeleteVersion() throws IOException {
706
707 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
708
709
710 KeyValue delete = KeyValueTestUtil.create(
711 "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
712 memstore.delete(delete);
713
714 assertEquals(2, memstore.kvset.size());
715 assertEquals(delete, memstore.kvset.first());
716 }
717 public void testRetainsDeleteColumn() throws IOException {
718
719 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
720
721
722 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
723 KeyValue.Type.DeleteColumn, "dont-care");
724 memstore.delete(delete);
725
726 assertEquals(2, memstore.kvset.size());
727 assertEquals(delete, memstore.kvset.first());
728 }
729 public void testRetainsDeleteFamily() throws IOException {
730
731 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
732
733
734 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
735 KeyValue.Type.DeleteFamily, "dont-care");
736 memstore.delete(delete);
737
738 assertEquals(2, memstore.kvset.size());
739 assertEquals(delete, memstore.kvset.first());
740 }
741
742
743
744
745
746
747
748
749 public void testMultipleTimestamps() throws IOException {
750 long[] timestamps = new long[] {20,10,5,1};
751 Scan scan = new Scan();
752
753 for (long timestamp: timestamps)
754 addRows(memstore,timestamp);
755
756 scan.setTimeRange(0, 2);
757 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
758
759 scan.setTimeRange(20, 82);
760 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
761
762 scan.setTimeRange(10, 20);
763 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
764
765 scan.setTimeRange(8, 12);
766 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
767
768
769
770
771
772 }
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791 public void testUpsertMSLAB() throws Exception {
792 Configuration conf = HBaseConfiguration.create();
793 conf.setBoolean(MemStore.USEMSLAB_KEY, true);
794 memstore = new MemStore(conf, KeyValue.COMPARATOR);
795
796 int ROW_SIZE = 2048;
797 byte[] qualifier = new byte[ROW_SIZE - 4];
798
799 MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
800 for (int i = 0; i < 3; i++) { System.gc(); }
801 long usageBefore = bean.getHeapMemoryUsage().getUsed();
802
803 long size = 0;
804 long ts=0;
805
806 for (int newValue = 0; newValue < 1000; newValue++) {
807 for (int row = newValue; row < newValue + 1000; row++) {
808 byte[] rowBytes = Bytes.toBytes(row);
809 size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
810 }
811 }
812 System.out.println("Wrote " + ts + " vals");
813 for (int i = 0; i < 3; i++) { System.gc(); }
814 long usageAfter = bean.getHeapMemoryUsage().getUsed();
815 System.out.println("Memory used: " + (usageAfter - usageBefore)
816 + " (heapsize: " + memstore.heapSize() +
817 " size: " + size + ")");
818 }
819
820
821
822
823 private static byte [] makeQualifier(final int i1, final int i2){
824 return Bytes.toBytes(Integer.toString(i1) + ";" +
825 Integer.toString(i2));
826 }
827
828
829
830
831
832
833 public void testUpsertMemstoreSize() throws Exception {
834 Configuration conf = HBaseConfiguration.create();
835 memstore = new MemStore(conf, KeyValue.COMPARATOR);
836 long oldSize = memstore.size.get();
837
838 List<Cell> l = new ArrayList<Cell>();
839 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
840 KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
841 KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");
842
843 kv1.setMvccVersion(1); kv2.setMvccVersion(1);kv3.setMvccVersion(1);
844 l.add(kv1); l.add(kv2); l.add(kv3);
845
846 this.memstore.upsert(l, 2);
847 long newSize = this.memstore.size.get();
848 assert(newSize > oldSize);
849
850 KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
851 kv4.setMvccVersion(1);
852 l.clear(); l.add(kv4);
853 this.memstore.upsert(l, 3);
854 assertEquals(newSize, this.memstore.size.get());
855
856 }
857
858
859
860
861
862
863
864
865
866
867
868 public void testUpdateToTimeOfOldestEdit() throws Exception {
869 try {
870 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
871 EnvironmentEdgeManager.injectEdge(edge);
872 MemStore memstore = new MemStore();
873 long t = memstore.timeOfOldestEdit();
874 assertEquals(t, Long.MAX_VALUE);
875
876
877 memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
878 t = memstore.timeOfOldestEdit();
879 assertTrue(t == 1234);
880
881
882 t = runSnapshot(memstore);
883
884
885 memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
886 t = memstore.timeOfOldestEdit();
887 assertTrue(t == 1234);
888 t = runSnapshot(memstore);
889
890
891 List<Cell> l = new ArrayList<Cell>();
892 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
893 kv1.setMvccVersion(100);
894 l.add(kv1);
895 memstore.upsert(l, 1000);
896 t = memstore.timeOfOldestEdit();
897 assertTrue(t == 1234);
898 } finally {
899 EnvironmentEdgeManager.reset();
900 }
901 }
902
903
904
905
906
907
908
909
910 public void testShouldFlush() throws Exception {
911 Configuration conf = new Configuration();
912 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
913 checkShouldFlush(conf, true);
914
915 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
916 checkShouldFlush(conf, false);
917 }
918
919 private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
920 try {
921 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
922 EnvironmentEdgeManager.injectEdge(edge);
923 HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf);
924 HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
925
926 Map<byte[], Store> stores = region.getStores();
927 assertTrue(stores.size() == 1);
928
929 Store s = stores.entrySet().iterator().next().getValue();
930 edge.setCurrentTimeMillis(1234);
931 s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
932 edge.setCurrentTimeMillis(1234 + 100);
933 assertTrue(region.shouldFlush() == false);
934 edge.setCurrentTimeMillis(1234 + 10000);
935 assertTrue(region.shouldFlush() == expected);
936 } finally {
937 EnvironmentEdgeManager.reset();
938 }
939 }
940
941 private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
942 long t = 1234;
943 @Override
944 public long currentTimeMillis() {
945 return t;
946 }
947 public void setCurrentTimeMillis(long t) {
948 this.t = t;
949 }
950 }
951
952
953
954
955
956
957
958 private int addRows(final MemStore hmc) {
959 return addRows(hmc, HConstants.LATEST_TIMESTAMP);
960 }
961
962
963
964
965
966
967
968 private int addRows(final MemStore hmc, final long ts) {
969 for (int i = 0; i < ROW_COUNT; i++) {
970 long timestamp = ts == HConstants.LATEST_TIMESTAMP?
971 System.currentTimeMillis(): ts;
972 for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
973 byte [] row = Bytes.toBytes(i);
974 byte [] qf = makeQualifier(i, ii);
975 hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
976 }
977 }
978 return ROW_COUNT;
979 }
980
981 private long runSnapshot(final MemStore hmc) throws UnexpectedException {
982
983 int oldHistorySize = hmc.getSnapshot().size();
984 hmc.snapshot();
985 KeyValueSkipListSet ss = hmc.getSnapshot();
986
987 assertTrue("History size has not increased", oldHistorySize < ss.size());
988 long t = memstore.timeOfOldestEdit();
989 assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
990 hmc.clearSnapshot(ss);
991 return t;
992 }
993
994 private void isExpectedRowWithoutTimestamps(final int rowIndex,
995 List<Cell> kvs) {
996 int i = 0;
997 for (Cell kv: kvs) {
998 byte[] expectedColname = makeQualifier(rowIndex, i++);
999 assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname));
1000
1001
1002
1003
1004 assertTrue("Content", CellUtil.matchingValue(kv, expectedColname));
1005 }
1006 }
1007
1008 private KeyValue getDeleteKV(byte [] row) {
1009 return new KeyValue(row, Bytes.toBytes("test_col"), null,
1010 HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
1011 }
1012
1013 private KeyValue getKV(byte [] row, byte [] value) {
1014 return new KeyValue(row, Bytes.toBytes("test_col"), null,
1015 HConstants.LATEST_TIMESTAMP, value);
1016 }
1017 private static void addRows(int count, final MemStore mem) {
1018 long nanos = System.nanoTime();
1019
1020 for (int i = 0 ; i < count ; i++) {
1021 if (i % 1000 == 0) {
1022
1023 System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
1024 nanos = System.nanoTime();
1025 }
1026 long timestamp = System.currentTimeMillis();
1027
1028 for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
1029 byte [] row = Bytes.toBytes(i);
1030 byte [] qf = makeQualifier(i, ii);
1031 mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
1032 }
1033 }
1034 }
1035
1036
1037 static void doScan(MemStore ms, int iteration) throws IOException {
1038 long nanos = System.nanoTime();
1039 KeyValueScanner s = ms.getScanners().get(0);
1040 s.seek(KeyValue.createFirstOnRow(new byte[]{}));
1041
1042 System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
1043 int cnt=0;
1044 while(s.next() != null) ++cnt;
1045
1046 System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
1047
1048 }
1049
1050 public static void main(String [] args) throws IOException {
1051 MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
1052 MemStore ms = new MemStore();
1053
1054 long n1 = System.nanoTime();
1055 addRows(25000, ms);
1056 System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
1057
1058
1059 System.out.println("foo");
1060
1061 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
1062
1063 for (int i = 0 ; i < 50 ; i++)
1064 doScan(ms, i);
1065
1066 }
1067
1068
1069
1070 }
1071