1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.rmi.UnexpectedException;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.List;
27 import java.util.NavigableSet;
28 import java.util.TreeSet;
29 import java.util.concurrent.atomic.AtomicReference;
30
31 import junit.framework.TestCase;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.KeyValue;
37 import org.apache.hadoop.hbase.KeyValueTestUtil;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.util.Bytes;
41
42 import com.google.common.base.Joiner;
43 import com.google.common.collect.Iterables;
44 import com.google.common.collect.Lists;
45
46
47 public class TestMemStore extends TestCase {
48 private final Log LOG = LogFactory.getLog(this.getClass());
49 private MemStore memstore;
50 private static final int ROW_COUNT = 10;
51 private static final int QUALIFIER_COUNT = ROW_COUNT;
52 private static final byte [] FAMILY = Bytes.toBytes("column");
53 private static final byte [] CONTENTS = Bytes.toBytes("contents");
54 private static final byte [] BASIC = Bytes.toBytes("basic");
55 private static final String CONTENTSTR = "contentstr";
56 private ReadWriteConsistencyControl rwcc;
57
58 @Override
59 public void setUp() throws Exception {
60 super.setUp();
61 this.rwcc = new ReadWriteConsistencyControl();
62 this.memstore = new MemStore();
63 }
64
65 public void testPutSameKey() {
66 byte [] bytes = Bytes.toBytes(getName());
67 KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
68 this.memstore.add(kv);
69 byte [] other = Bytes.toBytes("somethingelse");
70 KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
71 this.memstore.add(samekey);
72 KeyValue found = this.memstore.kvset.first();
73 assertEquals(1, this.memstore.kvset.size());
74 assertTrue(Bytes.toString(found.getValue()), Bytes.equals(samekey.getValue(),
75 found.getValue()));
76 }
77
78
79
80
81
82 public void testScanAcrossSnapshot() throws IOException {
83 int rowCount = addRows(this.memstore);
84 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
85 Scan scan = new Scan();
86 List<KeyValue> result = new ArrayList<KeyValue>();
87 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
88 StoreScanner s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
89 this.memstore.comparator, null, memstorescanners);
90 int count = 0;
91 try {
92 while (s.next(result)) {
93 LOG.info(result);
94 count++;
95
96 assertEquals(rowCount, result.size());
97 result.clear();
98 }
99 } finally {
100 s.close();
101 }
102 assertEquals(rowCount, count);
103 for (KeyValueScanner scanner : memstorescanners) {
104 scanner.close();
105 }
106
107 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
108 memstorescanners = this.memstore.getScanners();
109
110 s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
111 this.memstore.comparator, null, memstorescanners);
112 count = 0;
113 try {
114 while (s.next(result)) {
115 LOG.info(result);
116
117 assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
118 count++;
119
120 assertEquals(rowCount, result.size());
121 if (count == 2) {
122 this.memstore.snapshot();
123 LOG.info("Snapshotted");
124 }
125 result.clear();
126 }
127 } finally {
128 s.close();
129 }
130 assertEquals(rowCount, count);
131 for (KeyValueScanner scanner : memstorescanners) {
132 scanner.close();
133 }
134 memstorescanners = this.memstore.getScanners();
135
136 long ts = System.currentTimeMillis();
137 s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
138 this.memstore.comparator, null, memstorescanners);
139 count = 0;
140 int snapshotIndex = 5;
141 try {
142 while (s.next(result)) {
143 LOG.info(result);
144
145 assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
146
147 assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
148 count++;
149 if (count == snapshotIndex) {
150 this.memstore.snapshot();
151 this.memstore.clearSnapshot(this.memstore.getSnapshot());
152
153 addRows(this.memstore, ts);
154 LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
155 }
156 result.clear();
157 }
158 } finally {
159 s.close();
160 }
161 assertEquals(rowCount, count);
162 }
163
164
165
166
167
168 public void testScanAcrossSnapshot2() throws IOException {
169
170
171 final byte[] one = Bytes.toBytes(1);
172 final byte[] two = Bytes.toBytes(2);
173 final byte[] f = Bytes.toBytes("f");
174 final byte[] q = Bytes.toBytes("q");
175 final byte[] v = Bytes.toBytes(3);
176
177 final KeyValue kv1 = new KeyValue(one, f, q, v);
178 final KeyValue kv2 = new KeyValue(two, f, q, v);
179
180
181 this.memstore.add(kv1.clone());
182 this.memstore.add(kv2.clone());
183 verifyScanAcrossSnapshot2(kv1, kv2);
184
185
186 this.memstore.snapshot();
187 verifyScanAcrossSnapshot2(kv1, kv2);
188
189
190 this.memstore = new MemStore();
191 this.memstore.add(kv1.clone());
192 this.memstore.snapshot();
193 this.memstore.add(kv2.clone());
194 verifyScanAcrossSnapshot2(kv1, kv2);
195 }
196
197 private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
198 throws IOException {
199 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
200 List<KeyValueScanner> memstorescanners = this.memstore.getScanners();
201 assertEquals(1, memstorescanners.size());
202 final KeyValueScanner scanner = memstorescanners.get(0);
203 scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
204 assertEquals(kv1, scanner.next());
205 assertEquals(kv2, scanner.next());
206 assertNull(scanner.next());
207 }
208
209 private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
210 throws IOException {
211 scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
212 List<KeyValue> returned = Lists.newArrayList();
213
214 while (true) {
215 KeyValue next = scanner.next();
216 if (next == null) break;
217 returned.add(next);
218 }
219
220 assertTrue(
221 "Got:\n" + Joiner.on("\n").join(returned) +
222 "\nExpected:\n" + Joiner.on("\n").join(expected),
223 Iterables.elementsEqual(Arrays.asList(expected), returned));
224 assertNull(scanner.peek());
225 }
226
227 public void testMemstoreConcurrentControl() throws IOException {
228 final byte[] row = Bytes.toBytes(1);
229 final byte[] f = Bytes.toBytes("family");
230 final byte[] q1 = Bytes.toBytes("q1");
231 final byte[] q2 = Bytes.toBytes("q2");
232 final byte[] v = Bytes.toBytes("value");
233
234 ReadWriteConsistencyControl.WriteEntry w =
235 rwcc.beginMemstoreInsert();
236
237 KeyValue kv1 = new KeyValue(row, f, q1, v);
238 kv1.setMemstoreTS(w.getWriteNumber());
239 memstore.add(kv1);
240
241 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
242 KeyValueScanner s = this.memstore.getScanners().get(0);
243 assertScannerResults(s, new KeyValue[]{});
244
245 rwcc.completeMemstoreInsert(w);
246
247 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
248 s = this.memstore.getScanners().get(0);
249 assertScannerResults(s, new KeyValue[]{kv1});
250
251 w = rwcc.beginMemstoreInsert();
252 KeyValue kv2 = new KeyValue(row, f, q2, v);
253 kv2.setMemstoreTS(w.getWriteNumber());
254 memstore.add(kv2);
255
256 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
257 s = this.memstore.getScanners().get(0);
258 assertScannerResults(s, new KeyValue[]{kv1});
259
260 rwcc.completeMemstoreInsert(w);
261
262 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
263 s = this.memstore.getScanners().get(0);
264 assertScannerResults(s, new KeyValue[]{kv1, kv2});
265 }
266
267
268
269
270
271
272
273 public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
274 final byte[] row = Bytes.toBytes(1);
275 final byte[] f = Bytes.toBytes("family");
276 final byte[] q1 = Bytes.toBytes("q1");
277 final byte[] q2 = Bytes.toBytes("q2");
278 final byte[] v1 = Bytes.toBytes("value1");
279 final byte[] v2 = Bytes.toBytes("value2");
280
281
282 ReadWriteConsistencyControl.WriteEntry w =
283 rwcc.beginMemstoreInsert();
284
285 KeyValue kv11 = new KeyValue(row, f, q1, v1);
286 kv11.setMemstoreTS(w.getWriteNumber());
287 memstore.add(kv11);
288
289 KeyValue kv12 = new KeyValue(row, f, q2, v1);
290 kv12.setMemstoreTS(w.getWriteNumber());
291 memstore.add(kv12);
292 rwcc.completeMemstoreInsert(w);
293
294
295 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
296 KeyValueScanner s = this.memstore.getScanners().get(0);
297 assertScannerResults(s, new KeyValue[]{kv11, kv12});
298
299
300 w = rwcc.beginMemstoreInsert();
301 KeyValue kv21 = new KeyValue(row, f, q1, v2);
302 kv21.setMemstoreTS(w.getWriteNumber());
303 memstore.add(kv21);
304
305 KeyValue kv22 = new KeyValue(row, f, q2, v2);
306 kv22.setMemstoreTS(w.getWriteNumber());
307 memstore.add(kv22);
308
309
310 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
311 s = this.memstore.getScanners().get(0);
312 assertScannerResults(s, new KeyValue[]{kv11, kv12});
313
314
315 rwcc.completeMemstoreInsert(w);
316
317
318
319
320 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
321 s = this.memstore.getScanners().get(0);
322 assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
323 }
324
325
326
327
328
329
330 public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
331 final byte[] row = Bytes.toBytes(1);
332 final byte[] f = Bytes.toBytes("family");
333 final byte[] q1 = Bytes.toBytes("q1");
334 final byte[] q2 = Bytes.toBytes("q2");
335 final byte[] v1 = Bytes.toBytes("value1");
336
337 ReadWriteConsistencyControl.WriteEntry w =
338 rwcc.beginMemstoreInsert();
339
340 KeyValue kv11 = new KeyValue(row, f, q1, v1);
341 kv11.setMemstoreTS(w.getWriteNumber());
342 memstore.add(kv11);
343
344 KeyValue kv12 = new KeyValue(row, f, q2, v1);
345 kv12.setMemstoreTS(w.getWriteNumber());
346 memstore.add(kv12);
347 rwcc.completeMemstoreInsert(w);
348
349
350 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
351 KeyValueScanner s = this.memstore.getScanners().get(0);
352 assertScannerResults(s, new KeyValue[]{kv11, kv12});
353
354
355 w = rwcc.beginMemstoreInsert();
356 KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
357 KeyValue.Type.DeleteColumn);
358 kvDel.setMemstoreTS(w.getWriteNumber());
359 memstore.add(kvDel);
360
361
362 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
363 s = this.memstore.getScanners().get(0);
364 assertScannerResults(s, new KeyValue[]{kv11, kv12});
365
366
367 rwcc.completeMemstoreInsert(w);
368
369
370 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
371 s = this.memstore.getScanners().get(0);
372 assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
373 }
374
375
376 private static class ReadOwnWritesTester extends Thread {
377 static final int NUM_TRIES = 1000;
378
379 final byte[] row;
380
381 final byte[] f = Bytes.toBytes("family");
382 final byte[] q1 = Bytes.toBytes("q1");
383
384 final ReadWriteConsistencyControl rwcc;
385 final MemStore memstore;
386
387 AtomicReference<Throwable> caughtException;
388
389
390 public ReadOwnWritesTester(int id,
391 MemStore memstore,
392 ReadWriteConsistencyControl rwcc,
393 AtomicReference<Throwable> caughtException)
394 {
395 this.rwcc = rwcc;
396 this.memstore = memstore;
397 this.caughtException = caughtException;
398 row = Bytes.toBytes(id);
399 }
400
401 public void run() {
402 try {
403 internalRun();
404 } catch (Throwable t) {
405 caughtException.compareAndSet(null, t);
406 }
407 }
408
409 private void internalRun() throws IOException {
410 for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
411 ReadWriteConsistencyControl.WriteEntry w =
412 rwcc.beginMemstoreInsert();
413
414
415 byte[] v = Bytes.toBytes(i);
416
417 KeyValue kv = new KeyValue(row, f, q1, i, v);
418 kv.setMemstoreTS(w.getWriteNumber());
419 memstore.add(kv);
420 rwcc.completeMemstoreInsert(w);
421
422
423 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
424
425 KeyValueScanner s = this.memstore.getScanners().get(0);
426 s.seek(kv);
427
428 KeyValue ret = s.next();
429 assertNotNull("Didnt find own write at all", ret);
430 assertEquals("Didnt read own writes",
431 kv.getTimestamp(), ret.getTimestamp());
432 }
433 }
434 }
435
436 public void testReadOwnWritesUnderConcurrency() throws Throwable {
437
438 int NUM_THREADS = 8;
439
440 ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
441 AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
442
443 for (int i = 0; i < NUM_THREADS; i++) {
444 threads[i] = new ReadOwnWritesTester(i, memstore, rwcc, caught);
445 threads[i].start();
446 }
447
448 for (int i = 0; i < NUM_THREADS; i++) {
449 threads[i].join();
450 }
451
452 if (caught.get() != null) {
453 throw caught.get();
454 }
455 }
456
457
458
459
460
461 public void testSnapshotting() throws IOException {
462 final int snapshotCount = 5;
463
464 for (int i = 0; i < snapshotCount; i++) {
465 addRows(this.memstore);
466 runSnapshot(this.memstore);
467 KeyValueSkipListSet ss = this.memstore.getSnapshot();
468 assertEquals("History not being cleared", 0, ss.size());
469 }
470 }
471
472 public void testMultipleVersionsSimple() throws Exception {
473 MemStore m = new MemStore(KeyValue.COMPARATOR);
474 byte [] row = Bytes.toBytes("testRow");
475 byte [] family = Bytes.toBytes("testFamily");
476 byte [] qf = Bytes.toBytes("testQualifier");
477 long [] stamps = {1,2,3};
478 byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
479 Bytes.toBytes("value2")};
480 KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
481 KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
482 KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
483
484 m.add(key0);
485 m.add(key1);
486 m.add(key2);
487
488 assertTrue("Expected memstore to hold 3 values, actually has " +
489 m.kvset.size(), m.kvset.size() == 3);
490 }
491
492 public void testBinary() throws IOException {
493 MemStore mc = new MemStore(KeyValue.ROOT_COMPARATOR);
494 final int start = 43;
495 final int end = 46;
496 for (int k = start; k <= end; k++) {
497 byte [] kk = Bytes.toBytes(k);
498 byte [] row =
499 Bytes.toBytes(".META.,table," + Bytes.toString(kk) + ",1," + k);
500 KeyValue key = new KeyValue(row, CONTENTS, BASIC,
501 System.currentTimeMillis(),
502 (CONTENTSTR + k).getBytes(HConstants.UTF8_ENCODING));
503 mc.add(key);
504 System.out.println(key);
505
506
507
508
509
510 }
511 int index = start;
512 for (KeyValue kv: mc.kvset) {
513 System.out.println(kv);
514 byte [] b = kv.getRow();
515
516 String str = Bytes.toString(b, 13, 4);
517 byte [] bb = Bytes.toBytes(index);
518 String bbStr = Bytes.toString(bb);
519 assertEquals(str, bbStr);
520 index++;
521 }
522 }
523
524
525
526
527
528
529
530
531 public void testGetNextRow() throws Exception {
532 addRows(this.memstore);
533
534 Thread.sleep(1);
535 addRows(this.memstore);
536 KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
537 assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
538 new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
539 for (int i = 0; i < ROW_COUNT; i++) {
540 KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
541 System.currentTimeMillis()));
542 if (i + 1 == ROW_COUNT) {
543 assertEquals(nr, null);
544 } else {
545 assertTrue(KeyValue.COMPARATOR.compareRows(nr,
546 new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
547 }
548 }
549
550 for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
551 InternalScanner scanner =
552 new StoreScanner(new Scan(Bytes.toBytes(startRowId)), FAMILY,
553 Integer.MAX_VALUE, this.memstore.comparator, null,
554 memstore.getScanners());
555 List<KeyValue> results = new ArrayList<KeyValue>();
556 for (int i = 0; scanner.next(results); i++) {
557 int rowId = startRowId + i;
558 assertTrue("Row name",
559 KeyValue.COMPARATOR.compareRows(results.get(0),
560 Bytes.toBytes(rowId)) == 0);
561 assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
562 List<KeyValue> row = new ArrayList<KeyValue>();
563 for (KeyValue kv : results) {
564 row.add(kv);
565 }
566 isExpectedRowWithoutTimestamps(rowId, row);
567
568 results.clear();
569 }
570 }
571 }
572
573 public void testGet_memstoreAndSnapShot() throws IOException {
574 byte [] row = Bytes.toBytes("testrow");
575 byte [] fam = Bytes.toBytes("testfamily");
576 byte [] qf1 = Bytes.toBytes("testqualifier1");
577 byte [] qf2 = Bytes.toBytes("testqualifier2");
578 byte [] qf3 = Bytes.toBytes("testqualifier3");
579 byte [] qf4 = Bytes.toBytes("testqualifier4");
580 byte [] qf5 = Bytes.toBytes("testqualifier5");
581 byte [] val = Bytes.toBytes("testval");
582
583
584 memstore.add(new KeyValue(row, fam ,qf1, val));
585 memstore.add(new KeyValue(row, fam ,qf2, val));
586 memstore.add(new KeyValue(row, fam ,qf3, val));
587
588 memstore.snapshot();
589 assertEquals(3, memstore.snapshot.size());
590
591 assertEquals(0, memstore.kvset.size());
592 memstore.add(new KeyValue(row, fam ,qf4, val));
593 memstore.add(new KeyValue(row, fam ,qf5, val));
594 assertEquals(2, memstore.kvset.size());
595 }
596
597
598
599
600 public void testGetWithDelete() throws IOException {
601 byte [] row = Bytes.toBytes("testrow");
602 byte [] fam = Bytes.toBytes("testfamily");
603 byte [] qf1 = Bytes.toBytes("testqualifier");
604 byte [] val = Bytes.toBytes("testval");
605
606 long ts1 = System.nanoTime();
607 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
608 long ts2 = ts1 + 1;
609 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
610 long ts3 = ts2 +1;
611 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
612 memstore.add(put1);
613 memstore.add(put2);
614 memstore.add(put3);
615
616 assertEquals(3, memstore.kvset.size());
617
618 KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
619 memstore.delete(del2);
620
621 List<KeyValue> expected = new ArrayList<KeyValue>();
622 expected.add(put3);
623 expected.add(del2);
624 expected.add(put2);
625 expected.add(put1);
626
627 assertEquals(4, memstore.kvset.size());
628 int i = 0;
629 for(KeyValue kv : memstore.kvset) {
630 assertEquals(expected.get(i++), kv);
631 }
632 }
633
634 public void testGetWithDeleteColumn() throws IOException {
635 byte [] row = Bytes.toBytes("testrow");
636 byte [] fam = Bytes.toBytes("testfamily");
637 byte [] qf1 = Bytes.toBytes("testqualifier");
638 byte [] val = Bytes.toBytes("testval");
639
640 long ts1 = System.nanoTime();
641 KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
642 long ts2 = ts1 + 1;
643 KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
644 long ts3 = ts2 +1;
645 KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
646 memstore.add(put1);
647 memstore.add(put2);
648 memstore.add(put3);
649
650 assertEquals(3, memstore.kvset.size());
651
652 KeyValue del2 =
653 new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
654 memstore.delete(del2);
655
656 List<KeyValue> expected = new ArrayList<KeyValue>();
657 expected.add(put3);
658 expected.add(del2);
659 expected.add(put2);
660 expected.add(put1);
661
662
663 assertEquals(4, memstore.kvset.size());
664 int i = 0;
665 for (KeyValue kv: memstore.kvset) {
666 assertEquals(expected.get(i++), kv);
667 }
668 }
669
670
671 public void testGetWithDeleteFamily() throws IOException {
672 byte [] row = Bytes.toBytes("testrow");
673 byte [] fam = Bytes.toBytes("testfamily");
674 byte [] qf1 = Bytes.toBytes("testqualifier1");
675 byte [] qf2 = Bytes.toBytes("testqualifier2");
676 byte [] qf3 = Bytes.toBytes("testqualifier3");
677 byte [] val = Bytes.toBytes("testval");
678 long ts = System.nanoTime();
679
680 KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
681 KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
682 KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
683 KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
684
685 memstore.add(put1);
686 memstore.add(put2);
687 memstore.add(put3);
688 memstore.add(put4);
689
690 KeyValue del =
691 new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
692 memstore.delete(del);
693
694 List<KeyValue> expected = new ArrayList<KeyValue>();
695 expected.add(del);
696 expected.add(put1);
697 expected.add(put2);
698 expected.add(put4);
699 expected.add(put3);
700
701
702
703 assertEquals(5, memstore.kvset.size());
704 int i = 0;
705 for (KeyValue kv: memstore.kvset) {
706 assertEquals(expected.get(i++), kv);
707 }
708 }
709
710 public void testKeepDeleteInmemstore() {
711 byte [] row = Bytes.toBytes("testrow");
712 byte [] fam = Bytes.toBytes("testfamily");
713 byte [] qf = Bytes.toBytes("testqualifier");
714 byte [] val = Bytes.toBytes("testval");
715 long ts = System.nanoTime();
716 memstore.add(new KeyValue(row, fam, qf, ts, val));
717 KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
718 memstore.delete(delete);
719 assertEquals(2, memstore.kvset.size());
720 assertEquals(delete, memstore.kvset.first());
721 }
722
723 public void testRetainsDeleteVersion() throws IOException {
724
725 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
726
727
728 KeyValue delete = KeyValueTestUtil.create(
729 "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
730 memstore.delete(delete);
731
732 assertEquals(2, memstore.kvset.size());
733 assertEquals(delete, memstore.kvset.first());
734 }
735 public void testRetainsDeleteColumn() throws IOException {
736
737 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
738
739
740 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
741 KeyValue.Type.DeleteColumn, "dont-care");
742 memstore.delete(delete);
743
744 assertEquals(2, memstore.kvset.size());
745 assertEquals(delete, memstore.kvset.first());
746 }
747 public void testRetainsDeleteFamily() throws IOException {
748
749 memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
750
751
752 KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
753 KeyValue.Type.DeleteFamily, "dont-care");
754 memstore.delete(delete);
755
756 assertEquals(2, memstore.kvset.size());
757 assertEquals(delete, memstore.kvset.first());
758 }
759
760
761
762
763
764
765
766
767
768 public void testMultipleTimestamps() throws IOException {
769 long[] timestamps = new long[] {20,10,5,1};
770 Scan scan = new Scan();
771
772 for (long timestamp: timestamps)
773 addRows(memstore,timestamp);
774
775 scan.setTimeRange(0, 2);
776 assertTrue(memstore.shouldSeek(scan));
777
778 scan.setTimeRange(20, 82);
779 assertTrue(memstore.shouldSeek(scan));
780
781 scan.setTimeRange(10, 20);
782 assertTrue(memstore.shouldSeek(scan));
783
784 scan.setTimeRange(8, 12);
785 assertTrue(memstore.shouldSeek(scan));
786
787
788
789
790
791 }
792
793
794
795
796
797 private static byte [] makeQualifier(final int i1, final int i2){
798 return Bytes.toBytes(Integer.toString(i1) + ";" +
799 Integer.toString(i2));
800 }
801
802
803
804
805
806
807
808 private int addRows(final MemStore hmc) {
809 return addRows(hmc, HConstants.LATEST_TIMESTAMP);
810 }
811
812
813
814
815
816
817
818 private int addRows(final MemStore hmc, final long ts) {
819 for (int i = 0; i < ROW_COUNT; i++) {
820 long timestamp = ts == HConstants.LATEST_TIMESTAMP?
821 System.currentTimeMillis(): ts;
822 for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
823 byte [] row = Bytes.toBytes(i);
824 byte [] qf = makeQualifier(i, ii);
825 hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
826 }
827 }
828 return ROW_COUNT;
829 }
830
831 private void runSnapshot(final MemStore hmc) throws UnexpectedException {
832
833 int oldHistorySize = hmc.getSnapshot().size();
834 hmc.snapshot();
835 KeyValueSkipListSet ss = hmc.getSnapshot();
836
837 assertTrue("History size has not increased", oldHistorySize < ss.size());
838 hmc.clearSnapshot(ss);
839 }
840
841 private void isExpectedRowWithoutTimestamps(final int rowIndex,
842 List<KeyValue> kvs) {
843 int i = 0;
844 for (KeyValue kv: kvs) {
845 String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
846 String colnameStr = Bytes.toString(kv.getQualifier());
847 assertEquals("Column name", colnameStr, expectedColname);
848
849
850
851
852 String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
853 kv.getValueLength());
854 assertEquals("Content", colnameStr, colvalueStr);
855 }
856 }
857
858 private KeyValue getDeleteKV(byte [] row) {
859 return new KeyValue(row, Bytes.toBytes("test_col"), null,
860 HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
861 }
862
863 private KeyValue getKV(byte [] row, byte [] value) {
864 return new KeyValue(row, Bytes.toBytes("test_col"), null,
865 HConstants.LATEST_TIMESTAMP, value);
866 }
867 private static void addRows(int count, final MemStore mem) {
868 long nanos = System.nanoTime();
869
870 for (int i = 0 ; i < count ; i++) {
871 if (i % 1000 == 0) {
872
873 System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
874 nanos = System.nanoTime();
875 }
876 long timestamp = System.currentTimeMillis();
877
878 for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
879 byte [] row = Bytes.toBytes(i);
880 byte [] qf = makeQualifier(i, ii);
881 mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
882 }
883 }
884 }
885
886
887 static void doScan(MemStore ms, int iteration) throws IOException {
888 long nanos = System.nanoTime();
889 KeyValueScanner s = ms.getScanners().get(0);
890 s.seek(KeyValue.createFirstOnRow(new byte[]{}));
891
892 System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
893 int cnt=0;
894 while(s.next() != null) ++cnt;
895
896 System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos)/1000 + " for: " + cnt);
897
898 }
899
900 public static void main(String [] args) throws IOException {
901 ReadWriteConsistencyControl rwcc = new ReadWriteConsistencyControl();
902 MemStore ms = new MemStore();
903
904 long n1 = System.nanoTime();
905 addRows(25000, ms);
906 System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
907
908
909 System.out.println("foo");
910
911 ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
912
913 for (int i = 0 ; i < 50 ; i++)
914 doScan(ms, i);
915
916 }
917
918
919 }