1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.lang.management.ManagementFactory;
23 import java.lang.management.MemoryMXBean;
24 import java.rmi.UnexpectedException;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.concurrent.atomic.AtomicReference;
30
31 import junit.framework.TestCase;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.hbase.*;
37 import org.apache.hadoop.hbase.client.Scan;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.EnvironmentEdge;
40 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
41
42 import com.google.common.base.Joiner;
43 import com.google.common.collect.Iterables;
44 import com.google.common.collect.Lists;
45 import org.junit.experimental.categories.Category;
46
47
48 @Category(MediumTests.class)
49 public class TestMemStore extends TestCase {
50 private final Log LOG = LogFactory.getLog(this.getClass());
51 private MemStore memstore;
52 private static final int ROW_COUNT = 10;
53 private static final int QUALIFIER_COUNT = ROW_COUNT;
54 private static final byte [] FAMILY = Bytes.toBytes("column");
55 private static final byte [] CONTENTS = Bytes.toBytes("contents");
56 private static final byte [] BASIC = Bytes.toBytes("basic");
57 private static final String CONTENTSTR = "contentstr";
58 private MultiVersionConsistencyControl mvcc;
59
60 @Override
61 public void setUp() throws Exception {
62 super.setUp();
63 this.mvcc = new MultiVersionConsistencyControl();
64 this.memstore = new MemStore();
65 }
66
67 public void testPutSameKey() {
68 byte [] bytes = Bytes.toBytes(getName());
69 KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
70 this.memstore.add(kv);
71 byte [] other = Bytes.toBytes("somethingelse");
72 KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
73 this.memstore.add(samekey);
74 KeyValue found = this.memstore.kvset.first();
75 assertEquals(1, this.memstore.kvset.size());
76 assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(),
77 found.getValue()));
78 }
79
80
81
82
83
84 public void testScanAcrossSnapshot() throws IOException {
85 int rowCount = addRows(this.memstore);
86 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
87 Scan scan = new Scan();
88 List<KeyValue> result = new ArrayList<KeyValue>();
89 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
90 ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false,
91 0, this.memstore.comparator);
92 ScanType scanType = ScanType.USER_SCAN;
93 StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
94 int count = 0;
95 try {
96 while (s.next(result)) {
97 LOG.info(result);
98 count++;
99
100 assertEquals(rowCount, result.size());
101 result.clear();
102 }
103 } finally {
104 s.close();
105 }
106 assertEquals(rowCount, count);
107 for (KeyValueScanner scanner : memstorescanners) {
108 scanner.close();
109 }
110
111 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
112 memstorescanners = this.memstore.getScanners();
113
114 s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
115 count = 0;
116 try {
117 while (s.next(result)) {
118 LOG.info(result);
119
120 assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
121 count++;
122
123 assertEquals(rowCount, result.size());
124 if (count == 2) {
125 this.memstore.snapshot();
126 LOG.info("Snapshotted");
127 }
128 result.clear();
129 }
130 } finally {
131 s.close();
132 }
133 assertEquals(rowCount, count);
134 for (KeyValueScanner scanner : memstorescanners) {
135 scanner.close();
136 }
137 memstorescanners = this.memstore.getScanners();
138
139 long ts = System.currentTimeMillis();
140 s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
141 count = 0;
142 int snapshotIndex = 5;
143 try {
144 while (s.next(result)) {
145 LOG.info(result);
146
147 assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
148
149 assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
150 count++;
151 if (count == snapshotIndex) {
152 this.memstore.snapshot();
153 this.memstore.clearSnapshot(this.memstore.getSnapshot());
154
155 addRows(this.memstore, ts);
156 LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
157 }
158 result.clear();
159 }
160 } finally {
161 s.close();
162 }
163 assertEquals(rowCount, count);
164 }
165
166
167
168
169
170
171 public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
172
173
174 final byte[] one = Bytes.toBytes(1);
175 final byte[] two = Bytes.toBytes(2);
176 final byte[] f = Bytes.toBytes("f");
177 final byte[] q = Bytes.toBytes("q");
178 final byte[] v = Bytes.toBytes(3);
179
180 final KeyValue kv1 = new KeyValue(one, f, q, v);
181 final KeyValue kv2 = new KeyValue(two, f, q, v);
182
183
184 this.memstore.add(kv1.clone());
185 this.memstore.add(kv2.clone());
186 verifyScanAcrossSnapshot2(kv1, kv2);
187
188
189 this.memstore.snapshot();
190 verifyScanAcrossSnapshot2(kv1, kv2);
191
192
193 this.memstore = new MemStore();
194 this.memstore.add(kv1.clone());
195 this.memstore.snapshot();
196 this.memstore.add(kv2.clone());
197 verifyScanAcrossSnapshot2(kv1, kv2);
198 }
199
200 private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
201 throws IOException {
202 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
203 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
204 assertEquals(1, memstorescanners.size());
205 final KeyValueScanner scanner = memstorescanners.get(0);
206 scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
207 assertEquals(kv1, scanner.next());
208 assertEquals(kv2, scanner.next());
209 assertNull(scanner.next());
210 }
211
212 private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
213 throws IOException {
214 scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
215 List<KeyValue> returned = Lists.newArrayList();
216
217 while (true) {
218 KeyValue next = scanner.next();
219 if (next == null) break;
220 returned.add(next);
221 }
222
223 assertTrue(
224 "Got:\n" + Joiner.on("\n").join(returned) +
225 "\nExpected:\n" + Joiner.on("\n").join(expected),
226 Iterables.elementsEqual(Arrays.asList(expected), returned));
227 assertNull(scanner.peek());
228 }
229
230 public void testMemstoreConcurrentControl() throws IOException {
231 final byte[] row = Bytes.toBytes(1);
232 final byte[] f = Bytes.toBytes("family");
233 final byte[] q1 = Bytes.toBytes("q1");
234 final byte[] q2 = Bytes.toBytes("q2");
235 final byte[] v = Bytes.toBytes("value");
236
237 MultiVersionConsistencyControl.WriteEntry w =
238 mvcc.beginMemstoreInsert();
239
240 KeyValue kv1 = new KeyValue(row, f, q1, v);
241 kv1.setMvccVersion(w.getWriteNumber());
242 memstore.add(kv1);
243
244 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
245 KeyValueScanner s = this.memstore.getScanners().get(0);
246 assertScannerResults(s, new KeyValue[]{});
247
248 mvcc.completeMemstoreInsert(w);
249
250 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
251 s = this.memstore.getScanners().get(0);
252 assertScannerResults(s, new KeyValue[]{kv1});
253
254 w = mvcc.beginMemstoreInsert();
255 KeyValue kv2 = new KeyValue(row, f, q2, v);
256 kv2.setMvccVersion(w.getWriteNumber());
257 memstore.add(kv2);
258
259 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
260 s = this.memstore.getScanners().get(0);
261 assertScannerResults(s, new KeyValue[]{kv1});
262
263 mvcc.completeMemstoreInsert(w);
264
265 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
266 s = this.memstore.getScanners().get(0);
267 assertScannerResults(s, new KeyValue[]{kv1, kv2});
268 }
269
270
271
272
273
274
275
276 public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
277 final byte[] row = Bytes.toBytes(1);
278 final byte[] f = Bytes.toBytes("family");
279 final byte[] q1 = Bytes.toBytes("q1");
280 final byte[] q2 = Bytes.toBytes("q2");
281 final byte[] v1 = Bytes.toBytes("value1");
282 final byte[] v2 = Bytes.toBytes("value2");
283
284
285 MultiVersionConsistencyControl.WriteEntry w =
286 mvcc.beginMemstoreInsert();
287
288 KeyValue kv11 = new KeyValue(row, f, q1, v1);
289 kv11.setMvccVersion(w.getWriteNumber());
290 memstore.add(kv11);
291
292 KeyValue kv12 = new KeyValue(row, f, q2, v1);
293 kv12.setMvccVersion(w.getWriteNumber());
294 memstore.add(kv12);
295 mvcc.completeMemstoreInsert(w);
296
297
298 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
299 KeyValueScanner s = this.memstore.getScanners().get(0);
300 assertScannerResults(s, new KeyValue[]{kv11, kv12});
301
302
303 w = mvcc.beginMemstoreInsert();
304 KeyValue kv21 = new KeyValue(row, f, q1, v2);
305 kv21.setMvccVersion(w.getWriteNumber());
306 memstore.add(kv21);
307
308 KeyValue kv22 = new KeyValue(row, f, q2, v2);
309 kv22.setMvccVersion(w.getWriteNumber());
310 memstore.add(kv22);
311
312
313 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
314 s = this.memstore.getScanners().get(0);
315 assertScannerResults(s, new KeyValue[]{kv11, kv12});
316
317
318 mvcc.completeMemstoreInsert(w);
319
320
321
322
323 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
324 s = this.memstore.getScanners().get(0);
325 assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
326 }
327
328
329
330
331
332
333 public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
334 final byte[] row = Bytes.toBytes(1);
335 final byte[] f = Bytes.toBytes("family");
336 final byte[] q1 = Bytes.toBytes("q1");
337 final byte[] q2 = Bytes.toBytes("q2");
338 final byte[] v1 = Bytes.toBytes("value1");
339
340 MultiVersionConsistencyControl.WriteEntry w =
341 mvcc.beginMemstoreInsert();
342
343 KeyValue kv11 = new KeyValue(row, f, q1, v1);
344 kv11.setMvccVersion(w.getWriteNumber());
345 memstore.add(kv11);
346
347 KeyValue kv12 = new KeyValue(row, f, q2, v1);
348 kv12.setMvccVersion(w.getWriteNumber());
349 memstore.add(kv12);
350 mvcc.completeMemstoreInsert(w);
351
352
353 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
354 KeyValueScanner s = this.memstore.getScanners().get(0);
355 assertScannerResults(s, new KeyValue[]{kv11, kv12});
356
357
358 w = mvcc.beginMemstoreInsert();
359 KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
360 KeyValue.Type.DeleteColumn);
361 kvDel.setMvccVersion(w.getWriteNumber());
362 memstore.add(kvDel);
363
364
365 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
366 s = this.memstore.getScanners().get(0);
367 assertScannerResults(s, new KeyValue[]{kv11, kv12});
368
369
370 mvcc.completeMemstoreInsert(w);
371
372
373 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
374 s = this.memstore.getScanners().get(0);
375 assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
376 }
377
378
379 private static class ReadOwnWritesTester extends Thread {
380 static final int NUM_TRIES = 1000;
381
382 final byte[] row;
383
384 final byte[] f = Bytes.toBytes("family");
385 final byte[] q1 = Bytes.toBytes("q1");
386
387 final MultiVersionConsistencyControl mvcc;
388 final MemStore memstore;
389
390 AtomicReference<Throwable> caughtException;
391
392
393 public ReadOwnWritesTester(int id,
394 MemStore memstore,
395 MultiVersionConsistencyControl mvcc,
396 AtomicReference<Throwable> caughtException)
397 {
398 this.mvcc = mvcc;
399 this.memstore = memstore;
400 this.caughtException = caughtException;
401 row = Bytes.toBytes(id);
402 }
403
404 public void run() {
405 try {
406 internalRun();
407 } catch (Throwable t) {
408 caughtException.compareAndSet(null, t);
409 }
410 }
411
412 private void internalRun() throws IOException {
413 for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
414 MultiVersionConsistencyControl.WriteEntry w =
415 mvcc.beginMemstoreInsert();
416
417
418 byte[] v = Bytes.toBytes(i);
419
420 KeyValue kv = new KeyValue(row, f, q1, i, v);
421 kv.setMvccVersion(w.getWriteNumber());
422 memstore.add(kv);
423 mvcc.completeMemstoreInsert(w);
424
425
426 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
427
428 KeyValueScanner s = this.memstore.getScanners().get(0);
429 s.seek(kv);
430
431 KeyValue ret = s.next();
432 assertNotNull("Didnt find own write at all", ret);
433 assertEquals("Didnt read own writes",
434 kv.getTimestamp(), ret.getTimestamp());
435 }
436 }
437 }
438
439 public void testReadOwnWritesUnderConcurrency() throws Throwable {
440
441 int NUM_THREADS = 8;
442
443 ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
444 AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
445
446 for (int i = 0; i < NUM_THREADS; i++) {
447 threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
448 threads[i].start();
449 }
450
451 for (int i = 0; i < NUM_THREADS; i++) {
452 threads[i].join();
453 }
454
455 if (caught.get() != null) {
456 throw caught.get();
457 }
458 }
459
460
461
462
463
464 public void testSnapshotting() throws IOException {
465 final int snapshotCount = 5;
466
467 for (int i = 0; i < snapshotCount; i++) {
468 addRows(this.memstore);
469 runSnapshot(this.memstore);
470 KeyValueSkipListSet ss = this.memstore.getSnapshot();
471 assertEquals("History not being cleared", 0, ss.size());
472 }
473 }
474
475 public void testMultipleVersionsSimple() throws Exception {
476 MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
477 byte [] row = Bytes.toBytes("testRow");
478 byte [] family = Bytes.toBytes("testFamily");
479 byte [] qf = Bytes.toBytes("testQualifier");
480 long [] stamps = {1,2,3};
481 byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
482 Bytes.toBytes("value2")};
483 KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
484 KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
485 KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
486
487 m.add(key0);
488 m.add(key1);
489 m.add(key2);
490
491 assertTrue("Expected memstore to hold 3 values, actually has " +
492 m.kvset.size(), m.kvset.size() == 3);
493 }
494
495
496
497
498
499
500
501
502 public void testGetNextRow() throws Exception {
503 MultiVersionConsistencyControl.resetThreadReadPoint();
504 addRows(this.memstore);
505
506 Thread.sleep(1);
507 addRows(this.memstore);
508 KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
509 assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
510 new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
511 for (int i = 0; i < ROW_COUNT; i++) {
512 KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
513 System.currentTimeMillis()));
514 if (i + 1 == ROW_COUNT) {
515 assertEquals(nr, null);
516 } else {
517 assertTrue(KeyValue.COMPARATOR.compareRows(nr,
518 new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
519 }
520 }
521
522 for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
523 ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false,
524 0, this.memstore.comparator);
525 ScanType scanType = ScanType.USER_SCAN;
526 InternalScanner scanner = new StoreScanner(new Scan(
527 Bytes.toBytes(startRowId)), scanInfo, scanType, null,
528 memstore.getScanners());
529 List<KeyValue> results = new ArrayList<KeyValue>();
530 for (int i = 0; scanner.next(results); i++) {
531 int rowId = startRowId + i;
532 assertTrue("Row name",
533 KeyValue.COMPARATOR.compareRows(results.get(0),
534 Bytes.toBytes(rowId)) == 0);
535 assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
536 List<KeyValue> row = new ArrayList<KeyValue>();
537 for (KeyValue kv : results) {
538 row.add(kv);
539 }
540 isExpectedRowWithoutTimestamps(rowId, row);
541
542 results.clear();
543 }
544 }
545 }
546
547 public void testGet_memstoreAndSnapShot() throws IOException {
548 byte [] row = Bytes.toBytes("testrow");
549 byte [] fam = Bytes.toBytes("testfamily");
550 byte [] qf1 = Bytes.toBytes("testqualifier1");
551 byte [] qf2 = Bytes.toBytes("testqualifier2");
552 byte [] qf3 = Bytes.toBytes("testqualifier3");
553 byte [] qf4 = Bytes.toBytes("testqualifier4");
554 byte [] qf5 = Bytes.toBytes("testqualifier5");
555 byte [] val = Bytes.toBytes("testval");
556
557
558 memstore.add(new KeyValue(row, fam ,qf1, val));
559 memstore.add(new KeyValue(row, fam ,qf2, val));
560 memstore.add(new KeyValue(row, fam ,qf3, val));
561
562 memstore.snapshot();
563 assertEquals(3, memstore.snapshot.size());
564
565 assertEquals(0, memstore.kvset.size());
566 memstore.add(new KeyValue(row, fam ,qf4, val));
567 memstore.add(new KeyValue(row, fam ,qf5, val));
568 assertEquals(2, memstore.kvset.size());
569 }
570
571
572
573
574 public void testGetWithDelete() throws IOException {
575 byte [] row = Bytes.toBytes("testrow");
576 byte [] fam = Bytes.toBytes("testfamily");
577 byte [] qf1 = Bytes.toBytes("testqualifier");
578 byte [] val = Bytes.toBytes("testval");
579
580 long ts1 = System.nanoTime();
581 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
582 long ts2 = ts1 + 1;
583 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
584 long ts3 = ts2 +1;
585 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
586 memstore.add(put1);
587 memstore.add(put2);
588 memstore.add(put3);
589
590 assertEquals(3, memstore.kvset.size());
591
592 KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
593 memstore.delete(del2);
594
595 List<KeyValue> expected = new ArrayList<KeyValue>();
596 expected.add(put3);
597 expected.add(del2);
598 expected.add(put2);
599 expected.add(put1);
600
601 assertEquals(4, memstore.kvset.size());
602 int i = 0;
603 for(KeyValue kv : memstore.kvset) {
604 assertEquals(expected.get(i++), kv);
605 }
606 }
607
608 public void testGetWithDeleteColumn() throws IOException {
609 byte [] row = Bytes.toBytes("testrow");
610 byte [] fam = Bytes.toBytes("testfamily");
611 byte [] qf1 = Bytes.toBytes("testqualifier");
612 byte [] val = Bytes.toBytes("testval");
613
614 long ts1 = System.nanoTime();
615 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
616 long ts2 = ts1 + 1;
617 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
618 long ts3 = ts2 +1;
619 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
620 memstore.add(put1);
621 memstore.add(put2);
622 memstore.add(put3);
623
624 assertEquals(3, memstore.kvset.size());
625
626 KeyValue del2 =
627 new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
628 memstore.delete(del2);
629
630 List<KeyValue> expected = new ArrayList<KeyValue>();
631 expected.add(put3);
632 expected.add(del2);
633 expected.add(put2);
634 expected.add(put1);
635
636
637 assertEquals(4, memstore.kvset.size());
638 int i = 0;
639 for (KeyValue kv: memstore.kvset) {
640 assertEquals(expected.get(i++), kv);
641 }
642 }
643
644
645 public void testGetWithDeleteFamily() throws IOException {
646 byte [] row = Bytes.toBytes("testrow");
647 byte [] fam = Bytes.toBytes("testfamily");
648 byte [] qf1 = Bytes.toBytes("testqualifier1");
649 byte [] qf2 = Bytes.toBytes("testqualifier2");
650 byte [] qf3 = Bytes.toBytes("testqualifier3");
651 byte [] val = Bytes.toBytes("testval");
652 long ts = System.nanoTime();
653
654 KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
655 KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
656 KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
657 KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
658
659 memstore.add(put1);
660 memstore.add(put2);
661 memstore.add(put3);
662 memstore.add(put4);
663
664 KeyValue del =
665 new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
666 memstore.delete(del);
667
668 List<KeyValue> expected = new ArrayList<KeyValue>();
669 expected.add(del);
670 expected.add(put1);
671 expected.add(put2);
672 expected.add(put4);
673 expected.add(put3);
674
675
676
677 assertEquals(5, memstore.kvset.size());
678 int i = 0;
679 for (KeyValue kv: memstore.kvset) {
680 assertEquals(expected.get(i++), kv);
681 }
682 }
683
684 public void testKeepDeleteInmemstore() {
685 byte [] row = Bytes.toBytes("testrow");
686 byte [] fam = Bytes.toBytes("testfamily");
687 byte [] qf = Bytes.toBytes("testqualifier");
688 byte [] val = Bytes.toBytes("testval");
689 long ts = System.nanoTime();
690 memstore.add(new KeyValue(row, fam, qf, ts, val));
691 KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
692 memstore.delete(delete);
693 assertEquals(2, memstore.kvset.size());
694 assertEquals(delete, memstore.kvset.first());
695 }
696
697 public void testRetainsDeleteVersion() throws IOException {
698
699 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
700
701
702 KeyValue delete = KeyValueTestUtil.create(
703 "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
704 memstore.delete(delete);
705
706 assertEquals(2, memstore.kvset.size());
707 assertEquals(delete, memstore.kvset.first());
708 }
709 public void testRetainsDeleteColumn() throws IOException {
710
711 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
712
713
714 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
715 KeyValue.Type.DeleteColumn, "dont-care");
716 memstore.delete(delete);
717
718 assertEquals(2, memstore.kvset.size());
719 assertEquals(delete, memstore.kvset.first());
720 }
721 public void testRetainsDeleteFamily() throws IOException {
722
723 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
724
725
726 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
727 KeyValue.Type.DeleteFamily, "dont-care");
728 memstore.delete(delete);
729
730 assertEquals(2, memstore.kvset.size());
731 assertEquals(delete, memstore.kvset.first());
732 }
733
734
735
736
737
738
739
740
741 public void testMultipleTimestamps() throws IOException {
742 long[] timestamps = new long[] {20,10,5,1};
743 Scan scan = new Scan();
744
745 for (long timestamp: timestamps)
746 addRows(memstore,timestamp);
747
748 scan.setTimeRange(0, 2);
749 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
750
751 scan.setTimeRange(20, 82);
752 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
753
754 scan.setTimeRange(10, 20);
755 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
756
757 scan.setTimeRange(8, 12);
758 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
759
760
761
762
763
764 }
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783 public void testUpsertMSLAB() throws Exception {
784 Configuration conf = HBaseConfiguration.create();
785 conf.setBoolean(MemStore.USEMSLAB_KEY, true);
786 memstore = new MemStore(conf, KeyValue.COMPARATOR);
787
788 int ROW_SIZE = 2048;
789 byte[] qualifier = new byte[ROW_SIZE - 4];
790
791 MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
792 for (int i = 0; i < 3; i++) { System.gc(); }
793 long usageBefore = bean.getHeapMemoryUsage().getUsed();
794
795 long size = 0;
796 long ts=0;
797
798 for (int newValue = 0; newValue < 1000; newValue++) {
799 for (int row = newValue; row < newValue + 1000; row++) {
800 byte[] rowBytes = Bytes.toBytes(row);
801 size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
802 }
803 }
804 System.out.println("Wrote " + ts + " vals");
805 for (int i = 0; i < 3; i++) { System.gc(); }
806 long usageAfter = bean.getHeapMemoryUsage().getUsed();
807 System.out.println("Memory used: " + (usageAfter - usageBefore)
808 + " (heapsize: " + memstore.heapSize() +
809 " size: " + size + ")");
810 }
811
812
813
814
815 private static byte [] makeQualifier(final int i1, final int i2){
816 return Bytes.toBytes(Integer.toString(i1) + ";" +
817 Integer.toString(i2));
818 }
819
820
821
822
823
824
825 public void testUpsertMemstoreSize() throws Exception {
826 Configuration conf = HBaseConfiguration.create();
827 memstore = new MemStore(conf, KeyValue.COMPARATOR);
828 long oldSize = memstore.size.get();
829
830 List<Cell> l = new ArrayList<Cell>();
831 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
832 KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
833 KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");
834
835 kv1.setMvccVersion(1); kv2.setMvccVersion(1);kv3.setMvccVersion(1);
836 l.add(kv1); l.add(kv2); l.add(kv3);
837
838 this.memstore.upsert(l, 2);
839 long newSize = this.memstore.size.get();
840 assert(newSize > oldSize);
841
842 KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
843 kv4.setMvccVersion(1);
844 l.clear(); l.add(kv4);
845 this.memstore.upsert(l, 3);
846 assertEquals(newSize, this.memstore.size.get());
847
848 }
849
850
851
852
853
854
855
856
857
858
859
860 public void testUpdateToTimeOfOldestEdit() throws Exception {
861 try {
862 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
863 EnvironmentEdgeManager.injectEdge(edge);
864 MemStore memstore = new MemStore();
865 long t = memstore.timeOfOldestEdit();
866 assertEquals(t, Long.MAX_VALUE);
867
868
869 memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
870 t = memstore.timeOfOldestEdit();
871 assertTrue(t == 1234);
872
873
874 t = runSnapshot(memstore);
875
876
877 memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
878 t = memstore.timeOfOldestEdit();
879 assertTrue(t == 1234);
880 t = runSnapshot(memstore);
881
882
883 List<Cell> l = new ArrayList<Cell>();
884 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
885 kv1.setMvccVersion(100);
886 l.add(kv1);
887 memstore.upsert(l, 1000);
888 t = memstore.timeOfOldestEdit();
889 assertTrue(t == 1234);
890 } finally {
891 EnvironmentEdgeManager.reset();
892 }
893 }
894
895
896
897
898
899
900
901
902 public void testShouldFlush() throws Exception {
903 Configuration conf = new Configuration();
904 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
905 checkShouldFlush(conf, true);
906
907 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
908 checkShouldFlush(conf, false);
909 }
910
911 private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
912 try {
913 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
914 EnvironmentEdgeManager.injectEdge(edge);
915 HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf);
916 HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
917
918 Map<byte[], Store> stores = region.getStores();
919 assertTrue(stores.size() == 1);
920
921 Store s = stores.entrySet().iterator().next().getValue();
922 edge.setCurrentTimeMillis(1234);
923 s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
924 edge.setCurrentTimeMillis(1234 + 100);
925 assertTrue(region.shouldFlush() == false);
926 edge.setCurrentTimeMillis(1234 + 10000);
927 assertTrue(region.shouldFlush() == expected);
928 } finally {
929 EnvironmentEdgeManager.reset();
930 }
931 }
932
933 private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
934 long t = 1234;
935 @Override
936 public long currentTimeMillis() {
937 return t;
938 }
939 public void setCurrentTimeMillis(long t) {
940 this.t = t;
941 }
942 }
943
944
945
946
947
948
949
950 private int addRows(final MemStore hmc) {
951 return addRows(hmc, HConstants.LATEST_TIMESTAMP);
952 }
953
954
955
956
957
958
959
960 private int addRows(final MemStore hmc, final long ts) {
961 for (int i = 0; i < ROW_COUNT; i++) {
962 long timestamp = ts == HConstants.LATEST_TIMESTAMP?
963 System.currentTimeMillis(): ts;
964 for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
965 byte [] row = Bytes.toBytes(i);
966 byte [] qf = makeQualifier(i, ii);
967 hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
968 }
969 }
970 return ROW_COUNT;
971 }
972
973 private long runSnapshot(final MemStore hmc) throws UnexpectedException {
974
975 int oldHistorySize = hmc.getSnapshot().size();
976 hmc.snapshot();
977 KeyValueSkipListSet ss = hmc.getSnapshot();
978
979 assertTrue("History size has not increased", oldHistorySize < ss.size());
980 long t = memstore.timeOfOldestEdit();
981 assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
982 hmc.clearSnapshot(ss);
983 return t;
984 }
985
986 private void isExpectedRowWithoutTimestamps(final int rowIndex,
987 List<KeyValue> kvs) {
988 int i = 0;
989 for (KeyValue kv: kvs) {
990 String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
991 String colnameStr = Bytes.toString(kv.getQualifier());
992 assertEquals("Column name", colnameStr, expectedColname);
993
994
995
996
997 String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
998 kv.getValueLength());
999 assertEquals("Content", colnameStr, colvalueStr);
1000 }
1001 }
1002
1003 private KeyValue getDeleteKV(byte [] row) {
1004 return new KeyValue(row, Bytes.toBytes("test_col"), null,
1005 HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
1006 }
1007
1008 private KeyValue getKV(byte [] row, byte [] value) {
1009 return new KeyValue(row, Bytes.toBytes("test_col"), null,
1010 HConstants.LATEST_TIMESTAMP, value);
1011 }
1012 private static void addRows(int count, final MemStore mem) {
1013 long nanos = System.nanoTime();
1014
1015 for (int i = 0 ; i < count ; i++) {
1016 if (i % 1000 == 0) {
1017
1018 System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
1019 nanos = System.nanoTime();
1020 }
1021 long timestamp = System.currentTimeMillis();
1022
1023 for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
1024 byte [] row = Bytes.toBytes(i);
1025 byte [] qf = makeQualifier(i, ii);
1026 mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
1027 }
1028 }
1029 }
1030
1031
1032 static void doScan(MemStore ms, int iteration) throws IOException {
1033 long nanos = System.nanoTime();
1034 KeyValueScanner s = ms.getScanners().get(0);
1035 s.seek(KeyValue.createFirstOnRow(new byte[]{}));
1036
1037 System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
1038 int cnt=0;
1039 while(s.next() != null) ++cnt;
1040
1041 System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
1042
1043 }
1044
1045 public static void main(String [] args) throws IOException {
1046 MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
1047 MemStore ms = new MemStore();
1048
1049 long n1 = System.nanoTime();
1050 addRows(25000, ms);
1051 System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
1052
1053
1054 System.out.println("foo");
1055
1056 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
1057
1058 for (int i = 0 ; i < 50 ; i++)
1059 doScan(ms, i);
1060
1061 }
1062
1063
1064
1065 }
1066