1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.lang.management.ManagementFactory;
24 import java.lang.management.MemoryMXBean;
25 import java.rmi.UnexpectedException;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.Collections;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicReference;
32
33 import junit.framework.TestCase;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.*;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
41 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.EnvironmentEdge;
44 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45
46 import com.google.common.base.Joiner;
47 import com.google.common.collect.Iterables;
48 import com.google.common.collect.Lists;
49 import org.junit.experimental.categories.Category;
50
51
52 @Category(SmallTests.class)
53 public class TestMemStore extends TestCase {
54 private final Log LOG = LogFactory.getLog(this.getClass());
55 private MemStore memstore;
56 private static final int ROW_COUNT = 10;
57 private static final int QUALIFIER_COUNT = ROW_COUNT;
58 private static final byte [] FAMILY = Bytes.toBytes("column");
59 private static final byte [] CONTENTS = Bytes.toBytes("contents");
60 private static final byte [] BASIC = Bytes.toBytes("basic");
61 private static final String CONTENTSTR = "contentstr";
62 private MultiVersionConsistencyControl mvcc;
63
64 @Override
65 public void setUp() throws Exception {
66 super.setUp();
67 this.mvcc = new MultiVersionConsistencyControl();
68 this.memstore = new MemStore();
69 SchemaMetrics.setUseTableNameInTest(false);
70 }
71
72 public void testPutSameKey() {
73 byte [] bytes = Bytes.toBytes(getName());
74 KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
75 this.memstore.add(kv);
76 byte [] other = Bytes.toBytes("somethingelse");
77 KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
78 this.memstore.add(samekey);
79 KeyValue found = this.memstore.kvset.first();
80 assertEquals(1, this.memstore.kvset.size());
81 assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(),
82 found.getValue()));
83 }
84
85
86
87
88
89 public void testScanAcrossSnapshot() throws IOException {
90 int rowCount = addRows(this.memstore);
91 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
92 Scan scan = new Scan();
93 List<KeyValue> result = new ArrayList<KeyValue>();
94 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
95 ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false,
96 0, this.memstore.comparator);
97 ScanType scanType = ScanType.USER_SCAN;
98 StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
99 int count = 0;
100 try {
101 while (s.next(result)) {
102 LOG.info(result);
103 count++;
104
105 assertEquals(rowCount, result.size());
106 result.clear();
107 }
108 } finally {
109 s.close();
110 }
111 assertEquals(rowCount, count);
112 for (KeyValueScanner scanner : memstorescanners) {
113 scanner.close();
114 }
115
116 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
117 memstorescanners = this.memstore.getScanners();
118
119 s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
120 count = 0;
121 try {
122 while (s.next(result)) {
123 LOG.info(result);
124
125 assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
126 count++;
127
128 assertEquals(rowCount, result.size());
129 if (count == 2) {
130 this.memstore.snapshot();
131 LOG.info("Snapshotted");
132 }
133 result.clear();
134 }
135 } finally {
136 s.close();
137 }
138 assertEquals(rowCount, count);
139 for (KeyValueScanner scanner : memstorescanners) {
140 scanner.close();
141 }
142 memstorescanners = this.memstore.getScanners();
143
144 long ts = System.currentTimeMillis();
145 s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
146 count = 0;
147 int snapshotIndex = 5;
148 try {
149 while (s.next(result)) {
150 LOG.info(result);
151
152 assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
153
154 assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
155 count++;
156 if (count == snapshotIndex) {
157 this.memstore.snapshot();
158 this.memstore.clearSnapshot(this.memstore.getSnapshot());
159
160 addRows(this.memstore, ts);
161 LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
162 }
163 result.clear();
164 }
165 } finally {
166 s.close();
167 }
168 assertEquals(rowCount, count);
169 }
170
171
172
173
174
175 public void testScanAcrossSnapshot2() throws IOException {
176
177
178 final byte[] one = Bytes.toBytes(1);
179 final byte[] two = Bytes.toBytes(2);
180 final byte[] f = Bytes.toBytes("f");
181 final byte[] q = Bytes.toBytes("q");
182 final byte[] v = Bytes.toBytes(3);
183
184 final KeyValue kv1 = new KeyValue(one, f, q, v);
185 final KeyValue kv2 = new KeyValue(two, f, q, v);
186
187
188 this.memstore.add(kv1.clone());
189 this.memstore.add(kv2.clone());
190 verifyScanAcrossSnapshot2(kv1, kv2);
191
192
193 this.memstore.snapshot();
194 verifyScanAcrossSnapshot2(kv1, kv2);
195
196
197 this.memstore = new MemStore();
198 this.memstore.add(kv1.clone());
199 this.memstore.snapshot();
200 this.memstore.add(kv2.clone());
201 verifyScanAcrossSnapshot2(kv1, kv2);
202 }
203
204 private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
205 throws IOException {
206 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
207 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
208 assertEquals(1, memstorescanners.size());
209 final KeyValueScanner scanner = memstorescanners.get(0);
210 scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
211 assertEquals(kv1, scanner.next());
212 assertEquals(kv2, scanner.next());
213 assertNull(scanner.next());
214 }
215
216 private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
217 throws IOException {
218 scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
219 List<KeyValue> returned = Lists.newArrayList();
220
221 while (true) {
222 KeyValue next = scanner.next();
223 if (next == null) break;
224 returned.add(next);
225 }
226
227 assertTrue(
228 "Got:\n" + Joiner.on("\n").join(returned) +
229 "\nExpected:\n" + Joiner.on("\n").join(expected),
230 Iterables.elementsEqual(Arrays.asList(expected), returned));
231 assertNull(scanner.peek());
232 }
233
234 public void testMemstoreConcurrentControl() throws IOException {
235 final byte[] row = Bytes.toBytes(1);
236 final byte[] f = Bytes.toBytes("family");
237 final byte[] q1 = Bytes.toBytes("q1");
238 final byte[] q2 = Bytes.toBytes("q2");
239 final byte[] v = Bytes.toBytes("value");
240
241 MultiVersionConsistencyControl.WriteEntry w =
242 mvcc.beginMemstoreInsert();
243
244 KeyValue kv1 = new KeyValue(row, f, q1, v);
245 kv1.setMemstoreTS(w.getWriteNumber());
246 memstore.add(kv1);
247
248 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
249 KeyValueScanner s = this.memstore.getScanners().get(0);
250 assertScannerResults(s, new KeyValue[]{});
251
252 mvcc.completeMemstoreInsert(w);
253
254 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
255 s = this.memstore.getScanners().get(0);
256 assertScannerResults(s, new KeyValue[]{kv1});
257
258 w = mvcc.beginMemstoreInsert();
259 KeyValue kv2 = new KeyValue(row, f, q2, v);
260 kv2.setMemstoreTS(w.getWriteNumber());
261 memstore.add(kv2);
262
263 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
264 s = this.memstore.getScanners().get(0);
265 assertScannerResults(s, new KeyValue[]{kv1});
266
267 mvcc.completeMemstoreInsert(w);
268
269 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
270 s = this.memstore.getScanners().get(0);
271 assertScannerResults(s, new KeyValue[]{kv1, kv2});
272 }
273
274
275
276
277
278
279
280 public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
281 final byte[] row = Bytes.toBytes(1);
282 final byte[] f = Bytes.toBytes("family");
283 final byte[] q1 = Bytes.toBytes("q1");
284 final byte[] q2 = Bytes.toBytes("q2");
285 final byte[] v1 = Bytes.toBytes("value1");
286 final byte[] v2 = Bytes.toBytes("value2");
287
288
289 MultiVersionConsistencyControl.WriteEntry w =
290 mvcc.beginMemstoreInsert();
291
292 KeyValue kv11 = new KeyValue(row, f, q1, v1);
293 kv11.setMemstoreTS(w.getWriteNumber());
294 memstore.add(kv11);
295
296 KeyValue kv12 = new KeyValue(row, f, q2, v1);
297 kv12.setMemstoreTS(w.getWriteNumber());
298 memstore.add(kv12);
299 mvcc.completeMemstoreInsert(w);
300
301
302 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
303 KeyValueScanner s = this.memstore.getScanners().get(0);
304 assertScannerResults(s, new KeyValue[]{kv11, kv12});
305
306
307 w = mvcc.beginMemstoreInsert();
308 KeyValue kv21 = new KeyValue(row, f, q1, v2);
309 kv21.setMemstoreTS(w.getWriteNumber());
310 memstore.add(kv21);
311
312 KeyValue kv22 = new KeyValue(row, f, q2, v2);
313 kv22.setMemstoreTS(w.getWriteNumber());
314 memstore.add(kv22);
315
316
317 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
318 s = this.memstore.getScanners().get(0);
319 assertScannerResults(s, new KeyValue[]{kv11, kv12});
320
321
322 mvcc.completeMemstoreInsert(w);
323
324
325
326
327 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
328 s = this.memstore.getScanners().get(0);
329 assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
330 }
331
332
333
334
335
336
337 public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
338 final byte[] row = Bytes.toBytes(1);
339 final byte[] f = Bytes.toBytes("family");
340 final byte[] q1 = Bytes.toBytes("q1");
341 final byte[] q2 = Bytes.toBytes("q2");
342 final byte[] v1 = Bytes.toBytes("value1");
343
344 MultiVersionConsistencyControl.WriteEntry w =
345 mvcc.beginMemstoreInsert();
346
347 KeyValue kv11 = new KeyValue(row, f, q1, v1);
348 kv11.setMemstoreTS(w.getWriteNumber());
349 memstore.add(kv11);
350
351 KeyValue kv12 = new KeyValue(row, f, q2, v1);
352 kv12.setMemstoreTS(w.getWriteNumber());
353 memstore.add(kv12);
354 mvcc.completeMemstoreInsert(w);
355
356
357 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
358 KeyValueScanner s = this.memstore.getScanners().get(0);
359 assertScannerResults(s, new KeyValue[]{kv11, kv12});
360
361
362 w = mvcc.beginMemstoreInsert();
363 KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
364 KeyValue.Type.DeleteColumn);
365 kvDel.setMemstoreTS(w.getWriteNumber());
366 memstore.add(kvDel);
367
368
369 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
370 s = this.memstore.getScanners().get(0);
371 assertScannerResults(s, new KeyValue[]{kv11, kv12});
372
373
374 mvcc.completeMemstoreInsert(w);
375
376
377 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
378 s = this.memstore.getScanners().get(0);
379 assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
380 }
381
382
383 private static class ReadOwnWritesTester extends Thread {
384 static final int NUM_TRIES = 1000;
385
386 final byte[] row;
387
388 final byte[] f = Bytes.toBytes("family");
389 final byte[] q1 = Bytes.toBytes("q1");
390
391 final MultiVersionConsistencyControl mvcc;
392 final MemStore memstore;
393
394 AtomicReference<Throwable> caughtException;
395
396
397 public ReadOwnWritesTester(int id,
398 MemStore memstore,
399 MultiVersionConsistencyControl mvcc,
400 AtomicReference<Throwable> caughtException)
401 {
402 this.mvcc = mvcc;
403 this.memstore = memstore;
404 this.caughtException = caughtException;
405 row = Bytes.toBytes(id);
406 }
407
408 public void run() {
409 try {
410 internalRun();
411 } catch (Throwable t) {
412 caughtException.compareAndSet(null, t);
413 }
414 }
415
416 private void internalRun() throws IOException {
417 for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
418 MultiVersionConsistencyControl.WriteEntry w =
419 mvcc.beginMemstoreInsert();
420
421
422 byte[] v = Bytes.toBytes(i);
423
424 KeyValue kv = new KeyValue(row, f, q1, i, v);
425 kv.setMemstoreTS(w.getWriteNumber());
426 memstore.add(kv);
427 mvcc.completeMemstoreInsert(w);
428
429
430 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
431
432 KeyValueScanner s = this.memstore.getScanners().get(0);
433 s.seek(kv);
434
435 KeyValue ret = s.next();
436 assertNotNull("Didnt find own write at all", ret);
437 assertEquals("Didnt read own writes",
438 kv.getTimestamp(), ret.getTimestamp());
439 }
440 }
441 }
442
443 public void testReadOwnWritesUnderConcurrency() throws Throwable {
444
445 int NUM_THREADS = 8;
446
447 ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
448 AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
449
450 for (int i = 0; i < NUM_THREADS; i++) {
451 threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
452 threads[i].start();
453 }
454
455 for (int i = 0; i < NUM_THREADS; i++) {
456 threads[i].join();
457 }
458
459 if (caught.get() != null) {
460 throw caught.get();
461 }
462 }
463
464
465
466
467
468 public void testSnapshotting() throws IOException {
469 final int snapshotCount = 5;
470
471 for (int i = 0; i < snapshotCount; i++) {
472 addRows(this.memstore);
473 runSnapshot(this.memstore);
474 KeyValueSkipListSet ss = this.memstore.getSnapshot();
475 assertEquals("History not being cleared", 0, ss.size());
476 }
477 }
478
479 public void testMultipleVersionsSimple() throws Exception {
480 MemStore m = new MemStore(new Configuration(), KeyValue.COMPARATOR);
481 byte [] row = Bytes.toBytes("testRow");
482 byte [] family = Bytes.toBytes("testFamily");
483 byte [] qf = Bytes.toBytes("testQualifier");
484 long [] stamps = {1,2,3};
485 byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
486 Bytes.toBytes("value2")};
487 KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
488 KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
489 KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
490
491 m.add(key0);
492 m.add(key1);
493 m.add(key2);
494
495 assertTrue("Expected memstore to hold 3 values, actually has " +
496 m.kvset.size(), m.kvset.size() == 3);
497 }
498
499 public void testBinary() throws IOException {
500 MemStore mc = new MemStore(new Configuration(), KeyValue.ROOT_COMPARATOR);
501 final int start = 43;
502 final int end = 46;
503 for (int k = start; k <= end; k++) {
504 byte [] kk = Bytes.toBytes(k);
505 byte [] row =
506 Bytes.toBytes(".META.,table," + Bytes.toString(kk) + ",1," + k);
507 KeyValue key = new KeyValue(row, CONTENTS, BASIC,
508 System.currentTimeMillis(),
509 (CONTENTSTR + k).getBytes(HConstants.UTF8_ENCODING));
510 mc.add(key);
511 System.out.println(key);
512
513
514
515
516
517 }
518 int index = start;
519 for (KeyValue kv: mc.kvset) {
520 System.out.println(kv);
521 byte [] b = kv.getRow();
522
523 String str = Bytes.toString(b, 13, 4);
524 byte [] bb = Bytes.toBytes(index);
525 String bbStr = Bytes.toString(bb);
526 assertEquals(str, bbStr);
527 index++;
528 }
529 }
530
531
532
533
534
535
536
537
538 public void testGetNextRow() throws Exception {
539 MultiVersionConsistencyControl.resetThreadReadPoint();
540 addRows(this.memstore);
541
542 Thread.sleep(1);
543 addRows(this.memstore);
544 KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
545 assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
546 new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
547 for (int i = 0; i < ROW_COUNT; i++) {
548 KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
549 System.currentTimeMillis()));
550 if (i + 1 == ROW_COUNT) {
551 assertEquals(nr, null);
552 } else {
553 assertTrue(KeyValue.COMPARATOR.compareRows(nr,
554 new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
555 }
556 }
557
558 for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
559 ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false,
560 0, this.memstore.comparator);
561 ScanType scanType = ScanType.USER_SCAN;
562 InternalScanner scanner = new StoreScanner(new Scan(
563 Bytes.toBytes(startRowId)), scanInfo, scanType, null,
564 memstore.getScanners());
565 List<KeyValue> results = new ArrayList<KeyValue>();
566 for (int i = 0; scanner.next(results); i++) {
567 int rowId = startRowId + i;
568 assertTrue("Row name",
569 KeyValue.COMPARATOR.compareRows(results.get(0),
570 Bytes.toBytes(rowId)) == 0);
571 assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
572 List<KeyValue> row = new ArrayList<KeyValue>();
573 for (KeyValue kv : results) {
574 row.add(kv);
575 }
576 isExpectedRowWithoutTimestamps(rowId, row);
577
578 results.clear();
579 }
580 }
581 }
582
583 public void testGet_memstoreAndSnapShot() throws IOException {
584 byte [] row = Bytes.toBytes("testrow");
585 byte [] fam = Bytes.toBytes("testfamily");
586 byte [] qf1 = Bytes.toBytes("testqualifier1");
587 byte [] qf2 = Bytes.toBytes("testqualifier2");
588 byte [] qf3 = Bytes.toBytes("testqualifier3");
589 byte [] qf4 = Bytes.toBytes("testqualifier4");
590 byte [] qf5 = Bytes.toBytes("testqualifier5");
591 byte [] val = Bytes.toBytes("testval");
592
593
594 memstore.add(new KeyValue(row, fam ,qf1, val));
595 memstore.add(new KeyValue(row, fam ,qf2, val));
596 memstore.add(new KeyValue(row, fam ,qf3, val));
597
598 memstore.snapshot();
599 assertEquals(3, memstore.snapshot.size());
600
601 assertEquals(0, memstore.kvset.size());
602 memstore.add(new KeyValue(row, fam ,qf4, val));
603 memstore.add(new KeyValue(row, fam ,qf5, val));
604 assertEquals(2, memstore.kvset.size());
605 }
606
607
608
609
610 public void testGetWithDelete() throws IOException {
611 byte [] row = Bytes.toBytes("testrow");
612 byte [] fam = Bytes.toBytes("testfamily");
613 byte [] qf1 = Bytes.toBytes("testqualifier");
614 byte [] val = Bytes.toBytes("testval");
615
616 long ts1 = System.nanoTime();
617 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
618 long ts2 = ts1 + 1;
619 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
620 long ts3 = ts2 +1;
621 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
622 memstore.add(put1);
623 memstore.add(put2);
624 memstore.add(put3);
625
626 assertEquals(3, memstore.kvset.size());
627
628 KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
629 memstore.delete(del2);
630
631 List<KeyValue> expected = new ArrayList<KeyValue>();
632 expected.add(put3);
633 expected.add(del2);
634 expected.add(put2);
635 expected.add(put1);
636
637 assertEquals(4, memstore.kvset.size());
638 int i = 0;
639 for(KeyValue kv : memstore.kvset) {
640 assertEquals(expected.get(i++), kv);
641 }
642 }
643
644 public void testGetWithDeleteColumn() throws IOException {
645 byte [] row = Bytes.toBytes("testrow");
646 byte [] fam = Bytes.toBytes("testfamily");
647 byte [] qf1 = Bytes.toBytes("testqualifier");
648 byte [] val = Bytes.toBytes("testval");
649
650 long ts1 = System.nanoTime();
651 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
652 long ts2 = ts1 + 1;
653 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
654 long ts3 = ts2 +1;
655 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
656 memstore.add(put1);
657 memstore.add(put2);
658 memstore.add(put3);
659
660 assertEquals(3, memstore.kvset.size());
661
662 KeyValue del2 =
663 new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
664 memstore.delete(del2);
665
666 List<KeyValue> expected = new ArrayList<KeyValue>();
667 expected.add(put3);
668 expected.add(del2);
669 expected.add(put2);
670 expected.add(put1);
671
672
673 assertEquals(4, memstore.kvset.size());
674 int i = 0;
675 for (KeyValue kv: memstore.kvset) {
676 assertEquals(expected.get(i++), kv);
677 }
678 }
679
680
681 public void testGetWithDeleteFamily() throws IOException {
682 byte [] row = Bytes.toBytes("testrow");
683 byte [] fam = Bytes.toBytes("testfamily");
684 byte [] qf1 = Bytes.toBytes("testqualifier1");
685 byte [] qf2 = Bytes.toBytes("testqualifier2");
686 byte [] qf3 = Bytes.toBytes("testqualifier3");
687 byte [] val = Bytes.toBytes("testval");
688 long ts = System.nanoTime();
689
690 KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
691 KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
692 KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
693 KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
694
695 memstore.add(put1);
696 memstore.add(put2);
697 memstore.add(put3);
698 memstore.add(put4);
699
700 KeyValue del =
701 new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
702 memstore.delete(del);
703
704 List<KeyValue> expected = new ArrayList<KeyValue>();
705 expected.add(del);
706 expected.add(put1);
707 expected.add(put2);
708 expected.add(put4);
709 expected.add(put3);
710
711
712
713 assertEquals(5, memstore.kvset.size());
714 int i = 0;
715 for (KeyValue kv: memstore.kvset) {
716 assertEquals(expected.get(i++), kv);
717 }
718 }
719
720 public void testKeepDeleteInmemstore() {
721 byte [] row = Bytes.toBytes("testrow");
722 byte [] fam = Bytes.toBytes("testfamily");
723 byte [] qf = Bytes.toBytes("testqualifier");
724 byte [] val = Bytes.toBytes("testval");
725 long ts = System.nanoTime();
726 memstore.add(new KeyValue(row, fam, qf, ts, val));
727 KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
728 memstore.delete(delete);
729 assertEquals(2, memstore.kvset.size());
730 assertEquals(delete, memstore.kvset.first());
731 }
732
733 public void testRetainsDeleteVersion() throws IOException {
734
735 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
736
737
738 KeyValue delete = KeyValueTestUtil.create(
739 "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
740 memstore.delete(delete);
741
742 assertEquals(2, memstore.kvset.size());
743 assertEquals(delete, memstore.kvset.first());
744 }
745 public void testRetainsDeleteColumn() throws IOException {
746
747 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
748
749
750 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
751 KeyValue.Type.DeleteColumn, "dont-care");
752 memstore.delete(delete);
753
754 assertEquals(2, memstore.kvset.size());
755 assertEquals(delete, memstore.kvset.first());
756 }
757 public void testRetainsDeleteFamily() throws IOException {
758
759 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
760
761
762 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
763 KeyValue.Type.DeleteFamily, "dont-care");
764 memstore.delete(delete);
765
766 assertEquals(2, memstore.kvset.size());
767 assertEquals(delete, memstore.kvset.first());
768 }
769
770
771
772
773
774
775
776
777 public void testMultipleTimestamps() throws IOException {
778 long[] timestamps = new long[] {20,10,5,1};
779 Scan scan = new Scan();
780
781 for (long timestamp: timestamps)
782 addRows(memstore,timestamp);
783
784 scan.setTimeRange(0, 2);
785 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
786
787 scan.setTimeRange(20, 82);
788 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
789
790 scan.setTimeRange(10, 20);
791 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
792
793 scan.setTimeRange(8, 12);
794 assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
795
796
797
798
799
800 }
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819 public void testUpsertMSLAB() throws Exception {
820 Configuration conf = HBaseConfiguration.create();
821 conf.setBoolean(MemStore.USEMSLAB_KEY, true);
822 memstore = new MemStore(conf, KeyValue.COMPARATOR);
823
824 int ROW_SIZE = 2048;
825 byte[] qualifier = new byte[ROW_SIZE - 4];
826
827 MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
828 for (int i = 0; i < 3; i++) { System.gc(); }
829 long usageBefore = bean.getHeapMemoryUsage().getUsed();
830
831 long size = 0;
832 long ts=0;
833
834 for (int newValue = 0; newValue < 1000; newValue++) {
835 for (int row = newValue; row < newValue + 1000; row++) {
836 byte[] rowBytes = Bytes.toBytes(row);
837 size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
838 }
839 }
840 System.out.println("Wrote " + ts + " vals");
841 for (int i = 0; i < 3; i++) { System.gc(); }
842 long usageAfter = bean.getHeapMemoryUsage().getUsed();
843 System.out.println("Memory used: " + (usageAfter - usageBefore)
844 + " (heapsize: " + memstore.heapSize() +
845 " size: " + size + ")");
846 }
847
848
849
850
851 private static byte [] makeQualifier(final int i1, final int i2){
852 return Bytes.toBytes(Integer.toString(i1) + ";" +
853 Integer.toString(i2));
854 }
855
856
857
858
859
860
861 public void testUpsertMemstoreSize() throws Exception {
862 Configuration conf = HBaseConfiguration.create();
863 memstore = new MemStore(conf, KeyValue.COMPARATOR);
864 long oldSize = memstore.size.get();
865
866 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
867 this.memstore.upsert(Collections.singletonList(kv1));
868 long newSize = this.memstore.size.get();
869 assert(newSize > oldSize);
870
871 KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
872 this.memstore.upsert(Collections.singletonList(kv2));
873 assertEquals(newSize, this.memstore.size.get());
874 }
875
876
877
878
879
880
881
882
883
884
885
886 public void testUpdateToTimeOfOldestEdit() throws Exception {
887 try {
888 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
889 EnvironmentEdgeManager.injectEdge(edge);
890 MemStore memstore = new MemStore();
891 long t = memstore.timeOfOldestEdit();
892 assertEquals(t, Long.MAX_VALUE);
893
894
895 memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
896 t = memstore.timeOfOldestEdit();
897 assertTrue(t == 1234);
898
899
900 t = runSnapshot(memstore);
901
902
903 memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
904 t = memstore.timeOfOldestEdit();
905 assertTrue(t == 1234);
906 t = runSnapshot(memstore);
907
908
909 List<KeyValue> l = new ArrayList<KeyValue>();
910 KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
911 l.add(kv1);
912 memstore.upsert(l);
913 t = memstore.timeOfOldestEdit();
914 assertTrue(t == 1234);
915 } finally {
916 EnvironmentEdgeManager.reset();
917 }
918 }
919
920
921
922
923
924
925
926
927 public void testShouldFlush() throws Exception {
928 Configuration conf = new Configuration();
929 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
930 checkShouldFlush(conf, true);
931
932 conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
933 checkShouldFlush(conf, false);
934 }
935
936 private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
937 try {
938 EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
939 EnvironmentEdgeManager.injectEdge(edge);
940 HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf);
941 HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
942
943 Map<byte[], Store> stores = region.getStores();
944 assertTrue(stores.size() == 1);
945
946 Store s = stores.entrySet().iterator().next().getValue();
947 edge.setCurrentTimeMillis(1234);
948 s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
949 edge.setCurrentTimeMillis(1234 + 100);
950 assertTrue(region.shouldFlush() == false);
951 edge.setCurrentTimeMillis(1234 + 10000);
952 assertTrue(region.shouldFlush() == expected);
953 } finally {
954 EnvironmentEdgeManager.reset();
955 }
956 }
957
958 private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
959 long t = 1234;
960 @Override
961 public long currentTimeMillis() {
962 return t;
963 }
964 public void setCurrentTimeMillis(long t) {
965 this.t = t;
966 }
967 }
968
969
970
971
972
973
974 private int addRows(final MemStore hmc) {
975 return addRows(hmc, HConstants.LATEST_TIMESTAMP);
976 }
977
978
979
980
981
982
983
984 private int addRows(final MemStore hmc, final long ts) {
985 for (int i = 0; i < ROW_COUNT; i++) {
986 long timestamp = ts == HConstants.LATEST_TIMESTAMP?
987 System.currentTimeMillis(): ts;
988 for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
989 byte [] row = Bytes.toBytes(i);
990 byte [] qf = makeQualifier(i, ii);
991 hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
992 }
993 }
994 return ROW_COUNT;
995 }
996
997 private long runSnapshot(final MemStore hmc) throws UnexpectedException {
998
999 int oldHistorySize = hmc.getSnapshot().size();
1000 hmc.snapshot();
1001 KeyValueSkipListSet ss = hmc.getSnapshot();
1002
1003 assertTrue("History size has not increased", oldHistorySize < ss.size());
1004 long t = memstore.timeOfOldestEdit();
1005 assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
1006 hmc.clearSnapshot(ss);
1007 return t;
1008 }
1009
1010 private void isExpectedRowWithoutTimestamps(final int rowIndex,
1011 List<KeyValue> kvs) {
1012 int i = 0;
1013 for (KeyValue kv: kvs) {
1014 String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
1015 String colnameStr = Bytes.toString(kv.getQualifier());
1016 assertEquals("Column name", colnameStr, expectedColname);
1017
1018
1019
1020
1021 String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
1022 kv.getValueLength());
1023 assertEquals("Content", colnameStr, colvalueStr);
1024 }
1025 }
1026
1027 private KeyValue getDeleteKV(byte [] row) {
1028 return new KeyValue(row, Bytes.toBytes("test_col"), null,
1029 HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
1030 }
1031
1032 private KeyValue getKV(byte [] row, byte [] value) {
1033 return new KeyValue(row, Bytes.toBytes("test_col"), null,
1034 HConstants.LATEST_TIMESTAMP, value);
1035 }
1036 private static void addRows(int count, final MemStore mem) {
1037 long nanos = System.nanoTime();
1038
1039 for (int i = 0 ; i < count ; i++) {
1040 if (i % 1000 == 0) {
1041
1042 System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
1043 nanos = System.nanoTime();
1044 }
1045 long timestamp = System.currentTimeMillis();
1046
1047 for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
1048 byte [] row = Bytes.toBytes(i);
1049 byte [] qf = makeQualifier(i, ii);
1050 mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
1051 }
1052 }
1053 }
1054
1055
1056 static void doScan(MemStore ms, int iteration) throws IOException {
1057 long nanos = System.nanoTime();
1058 KeyValueScanner s = ms.getScanners().get(0);
1059 s.seek(KeyValue.createFirstOnRow(new byte[]{}));
1060
1061 System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
1062 int cnt=0;
1063 while(s.next() != null) ++cnt;
1064
1065 System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
1066
1067 }
1068
1069 public static void main(String [] args) throws IOException {
1070 MultiVersionConsistencyControl mvcc = new MultiVersionConsistencyControl();
1071 MemStore ms = new MemStore();
1072
1073 long n1 = System.nanoTime();
1074 addRows(25000, ms);
1075 System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
1076
1077
1078 System.out.println("foo");
1079
1080 MultiVersionConsistencyControl.resetThreadReadPoint(mvcc);
1081
1082 for (int i = 0 ; i < 50 ; i++)
1083 doScan(ms, i);
1084
1085 }
1086
1087
1088
1089 @org.junit.Rule
1090 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1091 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1092 }
1093