1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22
23 import java.io.DataInput;
24 import java.io.DataOutput;
25 import java.io.IOException;
26 import java.io.InterruptedIOException;
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.TreeMap;
33 import java.util.concurrent.atomic.AtomicBoolean;
34 import java.util.concurrent.atomic.AtomicInteger;
35 import java.util.concurrent.atomic.AtomicReference;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.fs.FSDataOutputStream;
41 import org.apache.hadoop.fs.FileSystem;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.hbase.DroppedSnapshotException;
44 import org.apache.hadoop.hbase.DoNotRetryIOException;
45 import org.apache.hadoop.hbase.HBaseConfiguration;
46 import org.apache.hadoop.hbase.HBaseTestCase;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HColumnDescriptor;
49 import org.apache.hadoop.hbase.HConstants;
50 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
51 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
52 import org.apache.hadoop.hbase.HRegionInfo;
53 import org.apache.hadoop.hbase.HTableDescriptor;
54 import org.apache.hadoop.hbase.KeyValue;
55 import org.apache.hadoop.hbase.MediumTests;
56 import org.apache.hadoop.hbase.MiniHBaseCluster;
57 import org.apache.hadoop.hbase.MultithreadedTestUtil;
58 import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
59 import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
60 import org.apache.hadoop.hbase.client.Append;
61 import org.apache.hadoop.hbase.client.Delete;
62 import org.apache.hadoop.hbase.client.Get;
63 import org.apache.hadoop.hbase.client.HTable;
64 import org.apache.hadoop.hbase.client.Increment;
65 import org.apache.hadoop.hbase.client.Put;
66 import org.apache.hadoop.hbase.client.Result;
67 import org.apache.hadoop.hbase.client.RowMutations;
68 import org.apache.hadoop.hbase.client.Scan;
69 import org.apache.hadoop.hbase.filter.BinaryComparator;
70 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
71 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
72 import org.apache.hadoop.hbase.filter.Filter;
73 import org.apache.hadoop.hbase.filter.FilterBase;
74 import org.apache.hadoop.hbase.filter.FilterList;
75 import org.apache.hadoop.hbase.filter.NullComparator;
76 import org.apache.hadoop.hbase.filter.PrefixFilter;
77 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
78 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
79 import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
80 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
81 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
82 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
83 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
84 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
85 import org.apache.hadoop.hbase.regionserver.wal.HLog;
86 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
87 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
88 import org.apache.hadoop.hbase.util.Bytes;
89 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
90 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
91 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
92 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
93 import org.apache.hadoop.hbase.util.Pair;
94 import org.apache.hadoop.hbase.util.PairOfSameType;
95 import org.apache.hadoop.hbase.util.Threads;
96 import org.junit.Assert;
97 import org.junit.Test;
98 import org.junit.experimental.categories.Category;
99 import org.mockito.Mockito;
100
101 import com.google.common.collect.Lists;
102
103
104
105
106
107
108
109
110 @Category(MediumTests.class)
111 @SuppressWarnings("deprecation")
112 public class TestHRegion extends HBaseTestCase {
113
114
115 static final Log LOG = LogFactory.getLog(TestHRegion.class);
116
117 private static final String COLUMN_FAMILY = "MyCF";
118
119 HRegion region = null;
120 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
121 private static final String DIR = TEST_UTIL.getDataTestDir("TestHRegion").toString();
122
123 private final int MAX_VERSIONS = 2;
124
125
126 protected final byte[] tableName = Bytes.toBytes("testtable");;
127 protected final byte[] qual1 = Bytes.toBytes("qual1");
128 protected final byte[] qual2 = Bytes.toBytes("qual2");
129 protected final byte[] qual3 = Bytes.toBytes("qual3");
130 protected final byte[] value1 = Bytes.toBytes("value1");
131 protected final byte[] value2 = Bytes.toBytes("value2");
132 protected final byte [] row = Bytes.toBytes("rowA");
133 protected final byte [] row2 = Bytes.toBytes("rowB");
134
135
136 private Map<String, Long> startingMetrics;
137
138
139
140
141 @Override
142 protected void setUp() throws Exception {
143 startingMetrics = SchemaMetrics.getMetricsSnapshot();
144 super.setUp();
145 }
146
147 @Override
148 protected void tearDown() throws Exception {
149 super.tearDown();
150 EnvironmentEdgeManagerTestHelper.reset();
151 SchemaMetrics.validateMetricChanges(startingMetrics);
152 }
153
154
155
156
157
158
159
160 public void testCompactionAffectedByScanners() throws Exception {
161 String method = "testCompactionAffectedByScanners";
162 byte[] tableName = Bytes.toBytes(method);
163 byte[] family = Bytes.toBytes("family");
164 this.region = initHRegion(tableName, method, conf, family);
165
166 Put put = new Put(Bytes.toBytes("r1"));
167 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
168 region.put(put);
169 region.flushcache();
170
171
172 Scan scan = new Scan();
173 scan.setMaxVersions(3);
174
175 RegionScanner scanner1 = region.getScanner(scan);
176
177 Delete delete = new Delete(Bytes.toBytes("r1"));
178 region.delete(delete, null, false);
179 region.flushcache();
180
181
182 RegionScanner scanner2 = region.getScanner(scan);
183
184 List<KeyValue> results = new ArrayList<KeyValue>();
185
186 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
187
188
189 region.compactStores(true);
190
191
192 RegionScanner scanner3 = region.getScanner(scan);
193
194
195 scanner1.next(results);
196 System.out.println(results);
197 assertEquals(1, results.size());
198
199 results.clear();
200 scanner2.next(results);
201 System.out.println(results);
202 assertEquals(0, results.size());
203
204 results.clear();
205 scanner3.next(results);
206 System.out.println(results);
207 assertEquals(0, results.size());
208 }
209
210 @Test
211 public void testToShowNPEOnRegionScannerReseek() throws Exception{
212 String method = "testToShowNPEOnRegionScannerReseek";
213 byte[] tableName = Bytes.toBytes(method);
214 byte[] family = Bytes.toBytes("family");
215 this.region = initHRegion(tableName, method, conf, family);
216
217 Put put = new Put(Bytes.toBytes("r1"));
218 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
219 region.put(put);
220 put = new Put(Bytes.toBytes("r2"));
221 put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
222 region.put(put);
223 region.flushcache();
224
225
226 Scan scan = new Scan();
227 scan.setMaxVersions(3);
228
229 RegionScanner scanner1 = region.getScanner(scan);
230
231 System.out.println("Smallest read point:" + region.getSmallestReadPoint());
232
233 region.compactStores(true);
234
235 scanner1.reseek(Bytes.toBytes("r2"));
236 List<KeyValue> results = new ArrayList<KeyValue>();
237 scanner1.next(results);
238 KeyValue keyValue = results.get(0);
239 Assert.assertTrue(Bytes.compareTo(keyValue.getRow(), Bytes.toBytes("r2")) == 0);
240 scanner1.close();
241 }
242
243 public void testSkipRecoveredEditsReplay() throws Exception {
244 String method = "testSkipRecoveredEditsReplay";
245 byte[] tableName = Bytes.toBytes(method);
246 byte[] family = Bytes.toBytes("family");
247 this.region = initHRegion(tableName, method, conf, family);
248 try {
249 Path regiondir = region.getRegionDir();
250 FileSystem fs = region.getFilesystem();
251 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
252
253 Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
254
255 long maxSeqId = 1050;
256 long minSeqId = 1000;
257
258 for (long i = minSeqId; i <= maxSeqId; i += 10) {
259 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
260 fs.create(recoveredEdits);
261 HLog.Writer writer = HLog.createWriter(fs, recoveredEdits, conf);
262
263 long time = System.nanoTime();
264 WALEdit edit = new WALEdit();
265 edit.add(new KeyValue(row, family, Bytes.toBytes(i),
266 time, KeyValue.Type.Put, Bytes.toBytes(i)));
267 writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
268 i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
269
270 writer.close();
271 }
272 MonitoredTask status = TaskMonitor.get().createStatus(method);
273 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
274 Bytes.BYTES_COMPARATOR);
275 for (Store store : region.getStores().values()) {
276 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
277 minSeqId - 1);
278 }
279 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
280 assertEquals(maxSeqId, seqId);
281 Get get = new Get(row);
282 Result result = region.get(get, null);
283 for (long i = minSeqId; i <= maxSeqId; i += 10) {
284 List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
285 assertEquals(1, kvs.size());
286 assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
287 }
288 } finally {
289 HRegion.closeHRegion(this.region);
290 this.region = null;
291 }
292 }
293
294 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
295 String method = "testSkipRecoveredEditsReplaySomeIgnored";
296 byte[] tableName = Bytes.toBytes(method);
297 byte[] family = Bytes.toBytes("family");
298 this.region = initHRegion(tableName, method, conf, family);
299 try {
300 Path regiondir = region.getRegionDir();
301 FileSystem fs = region.getFilesystem();
302 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
303
304 Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
305
306 long maxSeqId = 1050;
307 long minSeqId = 1000;
308
309 for (long i = minSeqId; i <= maxSeqId; i += 10) {
310 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
311 fs.create(recoveredEdits);
312 HLog.Writer writer = HLog.createWriter(fs, recoveredEdits, conf);
313
314 long time = System.nanoTime();
315 WALEdit edit = new WALEdit();
316 edit.add(new KeyValue(row, family, Bytes.toBytes(i),
317 time, KeyValue.Type.Put, Bytes.toBytes(i)));
318 writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
319 i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
320
321 writer.close();
322 }
323 long recoverSeqId = 1030;
324 MonitoredTask status = TaskMonitor.get().createStatus(method);
325 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
326 Bytes.BYTES_COMPARATOR);
327 for (Store store : region.getStores().values()) {
328 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
329 recoverSeqId - 1);
330 }
331 long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
332 assertEquals(maxSeqId, seqId);
333 Get get = new Get(row);
334 Result result = region.get(get, null);
335 for (long i = minSeqId; i <= maxSeqId; i += 10) {
336 List<KeyValue> kvs = result.getColumn(family, Bytes.toBytes(i));
337 if (i < recoverSeqId) {
338 assertEquals(0, kvs.size());
339 } else {
340 assertEquals(1, kvs.size());
341 assertEquals(Bytes.toBytes(i), kvs.get(0).getValue());
342 }
343 }
344 } finally {
345 HRegion.closeHRegion(this.region);
346 this.region = null;
347 }
348 }
349
350 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
351 String method = "testSkipRecoveredEditsReplayAllIgnored";
352 byte[] tableName = Bytes.toBytes(method);
353 byte[] family = Bytes.toBytes("family");
354 this.region = initHRegion(tableName, method, conf, family);
355 try {
356 Path regiondir = region.getRegionDir();
357 FileSystem fs = region.getFilesystem();
358
359 Path recoveredEditsDir = HLog.getRegionDirRecoveredEditsDir(regiondir);
360 for (int i = 1000; i < 1050; i += 10) {
361 Path recoveredEdits = new Path(
362 recoveredEditsDir, String.format("%019d", i));
363 FSDataOutputStream dos= fs.create(recoveredEdits);
364 dos.writeInt(i);
365 dos.close();
366 }
367 long minSeqId = 2000;
368 Path recoveredEdits = new Path(
369 recoveredEditsDir, String.format("%019d", minSeqId-1));
370 FSDataOutputStream dos= fs.create(recoveredEdits);
371 dos.close();
372
373 Map<byte[], Long> maxSeqIdInStores = new TreeMap<byte[], Long>(
374 Bytes.BYTES_COMPARATOR);
375 for (Store store : region.getStores().values()) {
376 maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
377 }
378 long seqId = region.replayRecoveredEditsIfAny(regiondir,
379 maxSeqIdInStores, null, null);
380 assertEquals(minSeqId, seqId);
381 } finally {
382 HRegion.closeHRegion(this.region);
383 this.region = null;
384 }
385 }
386
387 public void testGetWhileRegionClose() throws IOException {
388 Configuration hc = initSplit();
389 int numRows = 100;
390 byte [][] families = {fam1, fam2, fam3};
391
392
393 String method = this.getName();
394 this.region = initHRegion(tableName, method, hc, families);
395 try {
396
397 final int startRow = 100;
398 putData(startRow, numRows, qual1, families);
399 putData(startRow, numRows, qual2, families);
400 putData(startRow, numRows, qual3, families);
401
402 final AtomicBoolean done = new AtomicBoolean(false);
403 final AtomicInteger gets = new AtomicInteger(0);
404 GetTillDoneOrException [] threads = new GetTillDoneOrException[10];
405 try {
406
407 for (int i = 0; i < threads.length / 2; i++) {
408 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
409 done, gets);
410 threads[i].setDaemon(true);
411 threads[i].start();
412 }
413
414
415 this.region.closing.set(true);
416 for (int i = threads.length / 2; i < threads.length; i++) {
417 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow),
418 done, gets);
419 threads[i].setDaemon(true);
420 threads[i].start();
421 }
422 } finally {
423 if (this.region != null) {
424 this.region.close();
425 this.region.getLog().closeAndDelete();
426 }
427 }
428 done.set(true);
429 for (GetTillDoneOrException t: threads) {
430 try {
431 t.join();
432 } catch (InterruptedException e) {
433 e.printStackTrace();
434 }
435 if (t.e != null) {
436 LOG.info("Exception=" + t.e);
437 assertFalse("Found a NPE in " + t.getName(),
438 t.e instanceof NullPointerException);
439 }
440 }
441 } finally {
442 HRegion.closeHRegion(this.region);
443 this.region = null;
444 }
445 }
446
447
448
449
450
451 class GetTillDoneOrException extends Thread {
452 private final Get g;
453 private final AtomicBoolean done;
454 private final AtomicInteger count;
455 private Exception e;
456
457 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d,
458 final AtomicInteger c) {
459 super("getter." + i);
460 this.g = new Get(r);
461 this.done = d;
462 this.count = c;
463 }
464
465 @Override
466 public void run() {
467 while (!this.done.get()) {
468 try {
469 assertTrue(region.get(g, null).size() > 0);
470 this.count.incrementAndGet();
471 } catch (Exception e) {
472 this.e = e;
473 break;
474 }
475 }
476 }
477 }
478
479
480
481
482 public void testWeirdCacheBehaviour() throws Exception {
483 byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
484 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
485 Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
486 Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
487 this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
488 try {
489 String value = "this is the value";
490 String value2 = "this is some other value";
491 String keyPrefix1 = "prefix1";
492 String keyPrefix2 = "prefix2";
493 String keyPrefix3 = "prefix3";
494 putRows(this.region, 3, value, keyPrefix1);
495 putRows(this.region, 3, value, keyPrefix2);
496 putRows(this.region, 3, value, keyPrefix3);
497
498 putRows(this.region, 3, value2, keyPrefix1);
499 putRows(this.region, 3, value2, keyPrefix2);
500 putRows(this.region, 3, value2, keyPrefix3);
501 System.out.println("Checking values for key: " + keyPrefix1);
502 assertEquals("Got back incorrect number of rows from scan", 3,
503 getNumberOfRows(keyPrefix1, value2, this.region));
504 System.out.println("Checking values for key: " + keyPrefix2);
505 assertEquals("Got back incorrect number of rows from scan", 3,
506 getNumberOfRows(keyPrefix2, value2, this.region));
507 System.out.println("Checking values for key: " + keyPrefix3);
508 assertEquals("Got back incorrect number of rows from scan", 3,
509 getNumberOfRows(keyPrefix3, value2, this.region));
510 deleteColumns(this.region, value2, keyPrefix1);
511 deleteColumns(this.region, value2, keyPrefix2);
512 deleteColumns(this.region, value2, keyPrefix3);
513 System.out.println("Starting important checks.....");
514 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
515 0, getNumberOfRows(keyPrefix1, value2, this.region));
516 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
517 0, getNumberOfRows(keyPrefix2, value2, this.region));
518 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
519 0, getNumberOfRows(keyPrefix3, value2, this.region));
520 } finally {
521 HRegion.closeHRegion(this.region);
522 this.region = null;
523 }
524 }
525
526 public void testAppendWithReadOnlyTable() throws Exception {
527 byte[] TABLE = Bytes.toBytes("readOnlyTable");
528 this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
529 boolean exceptionCaught = false;
530 Append append = new Append(Bytes.toBytes("somerow"));
531 append.add(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
532 Bytes.toBytes("somevalue"));
533 try {
534 region.append(append, false);
535 } catch (IOException e) {
536 exceptionCaught = true;
537 } finally {
538 HRegion.closeHRegion(this.region);
539 this.region = null;
540 }
541 assertTrue(exceptionCaught == true);
542 }
543
544 public void testIncrWithReadOnlyTable() throws Exception {
545 byte[] TABLE = Bytes.toBytes("readOnlyTable");
546 this.region = initHRegion(TABLE, getName(), conf, true, Bytes.toBytes("somefamily"));
547 boolean exceptionCaught = false;
548 Increment inc = new Increment(Bytes.toBytes("somerow"));
549 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
550 try {
551 region.increment(inc, false);
552 } catch (IOException e) {
553 exceptionCaught = true;
554 } finally {
555 HRegion.closeHRegion(this.region);
556 this.region = null;
557 }
558 assertTrue(exceptionCaught == true);
559 }
560
561 private void deleteColumns(HRegion r, String value, String keyPrefix)
562 throws IOException {
563 InternalScanner scanner = buildScanner(keyPrefix, value, r);
564 int count = 0;
565 boolean more = false;
566 List<KeyValue> results = new ArrayList<KeyValue>();
567 do {
568 more = scanner.next(results);
569 if (results != null && !results.isEmpty())
570 count++;
571 else
572 break;
573 Delete delete = new Delete(results.get(0).getRow());
574 delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
575 r.delete(delete, null, false);
576 results.clear();
577 } while (more);
578 assertEquals("Did not perform correct number of deletes", 3, count);
579 }
580
581 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
582 InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
583 int numberOfResults = 0;
584 List<KeyValue> results = new ArrayList<KeyValue>();
585 boolean more = false;
586 do {
587 more = resultScanner.next(results);
588 if (results != null && !results.isEmpty()) numberOfResults++;
589 else break;
590 for (KeyValue kv: results) {
591 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue()));
592 }
593 results.clear();
594 } while(more);
595 return numberOfResults;
596 }
597
598 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
599 throws IOException {
600
601 FilterList allFilters = new FilterList();
602 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
603
604 SingleColumnValueFilter filter =
605 new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
606 Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
607 filter.setFilterIfMissing(true);
608 allFilters.addFilter(filter);
609 Scan scan = new Scan();
610 scan.addFamily(Bytes.toBytes("trans-blob"));
611 scan.addFamily(Bytes.toBytes("trans-type"));
612 scan.addFamily(Bytes.toBytes("trans-date"));
613 scan.addFamily(Bytes.toBytes("trans-tags"));
614 scan.addFamily(Bytes.toBytes("trans-group"));
615 scan.setFilter(allFilters);
616 return r.getScanner(scan);
617 }
618
619 private void putRows(HRegion r, int numRows, String value, String key)
620 throws IOException {
621 for (int i = 0; i < numRows; i++) {
622 String row = key + "_" + i
623 System.out.println(String.format("Saving row: %s, with value %s", row,
624 value));
625 Put put = new Put(Bytes.toBytes(row));
626 put.setWriteToWAL(false);
627 put.add(Bytes.toBytes("trans-blob"), null,
628 Bytes.toBytes("value for blob"));
629 put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
630 put.add(Bytes.toBytes("trans-date"), null,
631 Bytes.toBytes("20090921010101999"));
632 put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"),
633 Bytes.toBytes(value));
634 put.add(Bytes.toBytes("trans-group"), null,
635 Bytes.toBytes("adhocTransactionGroupId"));
636 r.put(put);
637 }
638 }
639
640 public void testFamilyWithAndWithoutColon() throws Exception {
641 byte [] b = Bytes.toBytes(getName());
642 byte [] cf = Bytes.toBytes(COLUMN_FAMILY);
643 this.region = initHRegion(b, getName(), conf, cf);
644 try {
645 Put p = new Put(b);
646 byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
647 p.add(cfwithcolon, cfwithcolon, cfwithcolon);
648 boolean exception = false;
649 try {
650 this.region.put(p);
651 } catch (NoSuchColumnFamilyException e) {
652 exception = true;
653 }
654 assertTrue(exception);
655 } finally {
656 HRegion.closeHRegion(this.region);
657 this.region = null;
658 }
659 }
660
661 @SuppressWarnings("unchecked")
662 public void testBatchPut() throws Exception {
663 byte[] b = Bytes.toBytes(getName());
664 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
665 byte[] qual = Bytes.toBytes("qual");
666 byte[] val = Bytes.toBytes("val");
667 this.region = initHRegion(b, getName(), conf, cf);
668 try {
669 HLog.getSyncTime();
670 assertEquals(0, HLog.getSyncTime().count);
671
672 LOG.info("First a batch put with all valid puts");
673 final Put[] puts = new Put[10];
674 for (int i = 0; i < 10; i++) {
675 puts[i] = new Put(Bytes.toBytes("row_" + i));
676 puts[i].add(cf, qual, val);
677 }
678
679 OperationStatus[] codes = this.region.put(puts);
680 assertEquals(10, codes.length);
681 for (int i = 0; i < 10; i++) {
682 assertEquals(OperationStatusCode.SUCCESS, codes[i]
683 .getOperationStatusCode());
684 }
685 assertEquals(1, HLog.getSyncTime().count);
686
687 LOG.info("Next a batch put with one invalid family");
688 puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
689 codes = this.region.put(puts);
690 assertEquals(10, codes.length);
691 for (int i = 0; i < 10; i++) {
692 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
693 OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
694 }
695 assertEquals(1, HLog.getSyncTime().count);
696
697 LOG.info("Next a batch put that has to break into two batches to avoid a lock");
698 Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
699
700 MultithreadedTestUtil.TestContext ctx =
701 new MultithreadedTestUtil.TestContext(conf);
702 final AtomicReference<OperationStatus[]> retFromThread =
703 new AtomicReference<OperationStatus[]>();
704 TestThread putter = new TestThread(ctx) {
705 @Override
706 public void doWork() throws IOException {
707 retFromThread.set(region.put(puts));
708 }
709 };
710 LOG.info("...starting put thread while holding lock");
711 ctx.addThread(putter);
712 ctx.startThreads();
713
714 LOG.info("...waiting for put thread to sync first time");
715 long startWait = System.currentTimeMillis();
716 while (HLog.getSyncTime().count == 0) {
717 Thread.sleep(100);
718 if (System.currentTimeMillis() - startWait > 10000) {
719 fail("Timed out waiting for thread to sync first minibatch");
720 }
721 }
722 LOG.info("...releasing row lock, which should let put thread continue");
723 region.releaseRowLock(lockedRow);
724 LOG.info("...joining on thread");
725 ctx.stop();
726 LOG.info("...checking that next batch was synced");
727 assertEquals(1, HLog.getSyncTime().count);
728 codes = retFromThread.get();
729 for (int i = 0; i < 10; i++) {
730 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
731 OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
732 }
733
734 LOG.info("Nexta, a batch put which uses an already-held lock");
735 lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
736 LOG.info("...obtained row lock");
737 List<Pair<Put, Integer>> putsAndLocks = Lists.newArrayList();
738 for (int i = 0; i < 10; i++) {
739 Pair<Put, Integer> pair = new Pair<Put, Integer>(puts[i], null);
740 if (i == 2) pair.setSecond(lockedRow);
741 putsAndLocks.add(pair);
742 }
743
744 codes = region.put(putsAndLocks.toArray(new Pair[0]));
745 LOG.info("...performed put");
746 for (int i = 0; i < 10; i++) {
747 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY :
748 OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
749 }
750
751 assertEquals(1, HLog.getSyncTime().count);
752
753
754 assertTrue(region.isRowLocked(lockedRow));
755 LOG.info("...releasing lock");
756 region.releaseRowLock(lockedRow);
757 } finally {
758 HRegion.closeHRegion(this.region);
759 this.region = null;
760 }
761 }
762
763 public void testBatchPutWithTsSlop() throws Exception {
764 byte[] b = Bytes.toBytes(getName());
765 byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
766 byte[] qual = Bytes.toBytes("qual");
767 byte[] val = Bytes.toBytes("val");
768 Configuration conf = HBaseConfiguration.create(this.conf);
769
770
771 conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
772 this.region = initHRegion(b, getName(), conf, cf);
773
774 try{
775 HLog.getSyncTime();
776 assertEquals(0, HLog.getSyncTime().count);
777
778 final Put[] puts = new Put[10];
779 for (int i = 0; i < 10; i++) {
780 puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
781 puts[i].add(cf, qual, val);
782 }
783
784 OperationStatus[] codes = this.region.put(puts);
785 assertEquals(10, codes.length);
786 for (int i = 0; i < 10; i++) {
787 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i]
788 .getOperationStatusCode());
789 }
790 assertEquals(0, HLog.getSyncTime().count);
791
792
793 } finally {
794 HRegion.closeHRegion(this.region);
795 this.region = null;
796 }
797
798 }
799
800
801
802
803 public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
804 byte [] tableName = Bytes.toBytes("testtable");
805 byte [] row1 = Bytes.toBytes("row1");
806 byte [] fam1 = Bytes.toBytes("fam1");
807 byte [] qf1 = Bytes.toBytes("qualifier");
808 byte [] emptyVal = new byte[] {};
809 byte [] val1 = Bytes.toBytes("value1");
810 byte [] val2 = Bytes.toBytes("value2");
811 Integer lockId = null;
812
813
814 String method = this.getName();
815 this.region = initHRegion(tableName, method, conf, fam1);
816 try {
817
818 Put put = new Put(row1);
819 put.add(fam1, qf1, emptyVal);
820
821
822 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
823 new BinaryComparator(emptyVal), put, lockId, true);
824 assertTrue(res);
825
826
827 put = new Put(row1);
828 put.add(fam1, qf1, val1);
829
830
831 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
832 new BinaryComparator(emptyVal), put, lockId, true);
833 assertTrue(res);
834
835
836 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
837 new BinaryComparator(emptyVal), put, lockId, true);
838 assertFalse(res);
839
840 Delete delete = new Delete(row1);
841 delete.deleteColumn(fam1, qf1);
842 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
843 new BinaryComparator(emptyVal), delete, lockId, true);
844 assertFalse(res);
845
846 put = new Put(row1);
847 put.add(fam1, qf1, val2);
848
849 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
850 new BinaryComparator(val1), put, lockId, true);
851 assertTrue(res);
852
853
854 delete = new Delete(row1);
855 delete.deleteColumn(fam1, qf1);
856 delete.deleteColumn(fam1, qf1);
857 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
858 new BinaryComparator(val2), delete, lockId, true);
859 assertTrue(res);
860
861 delete = new Delete(row1);
862 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
863 new BinaryComparator(emptyVal), delete, lockId, true);
864 assertTrue(res);
865
866
867 put = new Put(row1);
868 put.add(fam1, qf1, val1);
869
870 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
871 new NullComparator(), put, lockId, true);
872 assertTrue(res);
873 } finally {
874 HRegion.closeHRegion(this.region);
875 this.region = null;
876 }
877 }
878
879 public void testCheckAndMutate_WithWrongValue() throws IOException{
880 byte [] tableName = Bytes.toBytes("testtable");
881 byte [] row1 = Bytes.toBytes("row1");
882 byte [] fam1 = Bytes.toBytes("fam1");
883 byte [] qf1 = Bytes.toBytes("qualifier");
884 byte [] val1 = Bytes.toBytes("value1");
885 byte [] val2 = Bytes.toBytes("value2");
886 Integer lockId = null;
887
888
889 String method = this.getName();
890 this.region = initHRegion(tableName, method, conf, fam1);
891 try {
892
893 Put put = new Put(row1);
894 put.add(fam1, qf1, val1);
895 region.put(put);
896
897
898 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
899 new BinaryComparator(val2), put, lockId, true);
900 assertEquals(false, res);
901
902
903 Delete delete = new Delete(row1);
904 delete.deleteFamily(fam1);
905 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
906 new BinaryComparator(val2), delete, lockId, true);
907 assertEquals(false, res);
908 } finally {
909 HRegion.closeHRegion(this.region);
910 this.region = null;
911 }
912 }
913
914 public void testCheckAndMutate_WithCorrectValue() throws IOException{
915 byte [] tableName = Bytes.toBytes("testtable");
916 byte [] row1 = Bytes.toBytes("row1");
917 byte [] fam1 = Bytes.toBytes("fam1");
918 byte [] qf1 = Bytes.toBytes("qualifier");
919 byte [] val1 = Bytes.toBytes("value1");
920 Integer lockId = null;
921
922
923 String method = this.getName();
924 this.region = initHRegion(tableName, method, conf, fam1);
925 try {
926
927 Put put = new Put(row1);
928 put.add(fam1, qf1, val1);
929 region.put(put);
930
931
932 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
933 new BinaryComparator(val1), put, lockId, true);
934 assertEquals(true, res);
935
936
937 Delete delete = new Delete(row1);
938 delete.deleteColumn(fam1, qf1);
939 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
940 new BinaryComparator(val1), put, lockId, true);
941 assertEquals(true, res);
942 } finally {
943 HRegion.closeHRegion(this.region);
944 this.region = null;
945 }
946 }
947
948 public void testCheckAndPut_ThatPutWasWritten() throws IOException{
949 byte [] tableName = Bytes.toBytes("testtable");
950 byte [] row1 = Bytes.toBytes("row1");
951 byte [] fam1 = Bytes.toBytes("fam1");
952 byte [] fam2 = Bytes.toBytes("fam2");
953 byte [] qf1 = Bytes.toBytes("qualifier");
954 byte [] val1 = Bytes.toBytes("value1");
955 byte [] val2 = Bytes.toBytes("value2");
956 Integer lockId = null;
957
958 byte [][] families = {fam1, fam2};
959
960
961 String method = this.getName();
962 this.region = initHRegion(tableName, method, conf, families);
963 try {
964
965 Put put = new Put(row1);
966 put.add(fam1, qf1, val1);
967 region.put(put);
968
969
970 long ts = System.currentTimeMillis();
971 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
972 put = new Put(row1);
973 put.add(kv);
974
975
976 Store store = region.getStore(fam1);
977 store.memstore.kvset.size();
978
979 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
980 new BinaryComparator(val1), put, lockId, true);
981 assertEquals(true, res);
982 store.memstore.kvset.size();
983
984 Get get = new Get(row1);
985 get.addColumn(fam2, qf1);
986 KeyValue [] actual = region.get(get, null).raw();
987
988 KeyValue [] expected = {kv};
989
990 assertEquals(expected.length, actual.length);
991 for(int i=0; i<actual.length; i++) {
992 assertEquals(expected[i], actual[i]);
993 }
994 } finally {
995 HRegion.closeHRegion(this.region);
996 this.region = null;
997 }
998 }
999
1000 public void testCheckAndPut_wrongRowInPut() throws IOException {
1001 this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
1002 try {
1003 Put put = new Put(row2);
1004 put.add(fam1, qual1, value1);
1005 try {
1006 boolean res = region.checkAndMutate(row, fam1, qual1, CompareOp.EQUAL,
1007 new BinaryComparator(value2), put, null, false);
1008 fail();
1009 } catch (DoNotRetryIOException expected) {
1010
1011 }
1012 } finally {
1013 HRegion.closeHRegion(this.region);
1014 this.region = null;
1015 }
1016 }
1017
1018 public void testmutateRowsWithLocks_wrongCF() throws IOException {
1019 this.region = initHRegion(tableName, this.getName(), conf, fam1, fam2);
1020 try {
1021 Put put = new Put(row2);
1022 put.add(fam3, qual1, value1);
1023 RowMutations rm = new RowMutations(row2);
1024 rm.add(put);
1025 try {
1026 region.mutateRow(rm);
1027 fail();
1028 } catch (DoNotRetryIOException expected) {
1029
1030 LOG.debug("Caught expected exception: " + expected.getMessage());
1031 }
1032 } finally {
1033 HRegion.closeHRegion(this.region);
1034 this.region = null;
1035 }
1036 }
1037
1038 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException{
1039 byte [] tableName = Bytes.toBytes("testtable");
1040 byte [] row1 = Bytes.toBytes("row1");
1041 byte [] fam1 = Bytes.toBytes("fam1");
1042 byte [] fam2 = Bytes.toBytes("fam2");
1043 byte [] qf1 = Bytes.toBytes("qualifier1");
1044 byte [] qf2 = Bytes.toBytes("qualifier2");
1045 byte [] qf3 = Bytes.toBytes("qualifier3");
1046 byte [] val1 = Bytes.toBytes("value1");
1047 byte [] val2 = Bytes.toBytes("value2");
1048 byte [] val3 = Bytes.toBytes("value3");
1049 byte[] emptyVal = new byte[] { };
1050 Integer lockId = null;
1051
1052 byte [][] families = {fam1, fam2};
1053
1054
1055 String method = this.getName();
1056 this.region = initHRegion(tableName, method, conf, families);
1057 try {
1058
1059 Put put = new Put(row1);
1060 put.add(fam1, qf1, val1);
1061 region.put(put);
1062 Threads.sleep(2);
1063
1064 put = new Put(row1);
1065 put.add(fam1, qf1, val2);
1066 put.add(fam2, qf1, val3);
1067 put.add(fam2, qf2, val2);
1068 put.add(fam2, qf3, val1);
1069 put.add(fam1, qf3, val1);
1070 region.put(put);
1071
1072
1073 Delete delete = new Delete(row1);
1074 delete.deleteColumn(fam1, qf1);
1075 delete.deleteColumn(fam2, qf1);
1076 delete.deleteColumn(fam1, qf3);
1077 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1078 new BinaryComparator(val2), delete, lockId, true);
1079 assertEquals(true, res);
1080
1081 Get get = new Get(row1);
1082 get.addColumn(fam1, qf1);
1083 get.addColumn(fam1, qf3);
1084 get.addColumn(fam2, qf2);
1085 Result r = region.get(get, null);
1086 assertEquals(2, r.size());
1087 assertEquals(val1, r.getValue(fam1, qf1));
1088 assertEquals(val2, r.getValue(fam2, qf2));
1089
1090
1091 delete = new Delete(row1);
1092 delete.deleteFamily(fam2);
1093 res = region.checkAndMutate(row1, fam2, qf1, CompareOp.EQUAL,
1094 new BinaryComparator(emptyVal), delete, lockId, true);
1095 assertEquals(true, res);
1096
1097 get = new Get(row1);
1098 r = region.get(get, null);
1099 assertEquals(1, r.size());
1100 assertEquals(val1, r.getValue(fam1, qf1));
1101
1102
1103 delete = new Delete(row1);
1104 res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL,
1105 new BinaryComparator(val1), delete, lockId, true);
1106 assertEquals(true, res);
1107 get = new Get(row1);
1108 r = region.get(get, null);
1109 assertEquals(0, r.size());
1110 } finally {
1111 HRegion.closeHRegion(this.region);
1112 this.region = null;
1113 }
1114 }
1115
1116
1117
1118
1119 public void testDelete_multiDeleteColumn() throws IOException {
1120 byte [] tableName = Bytes.toBytes("testtable");
1121 byte [] row1 = Bytes.toBytes("row1");
1122 byte [] fam1 = Bytes.toBytes("fam1");
1123 byte [] qual = Bytes.toBytes("qualifier");
1124 byte [] value = Bytes.toBytes("value");
1125
1126 Put put = new Put(row1);
1127 put.add(fam1, qual, 1, value);
1128 put.add(fam1, qual, 2, value);
1129
1130 String method = this.getName();
1131 this.region = initHRegion(tableName, method, conf, fam1);
1132 try {
1133 region.put(put);
1134
1135
1136 Delete delete = new Delete(row1);
1137 delete.deleteColumn(fam1, qual);
1138 delete.deleteColumn(fam1, qual);
1139 region.delete(delete, null, false);
1140
1141 Get get = new Get(row1);
1142 get.addFamily(fam1);
1143 Result r = region.get(get, null);
1144 assertEquals(0, r.size());
1145 } finally {
1146 HRegion.closeHRegion(this.region);
1147 this.region = null;
1148 }
1149 }
1150
1151 public void testDelete_CheckFamily() throws IOException {
1152 byte [] tableName = Bytes.toBytes("testtable");
1153 byte [] row1 = Bytes.toBytes("row1");
1154 byte [] fam1 = Bytes.toBytes("fam1");
1155 byte [] fam2 = Bytes.toBytes("fam2");
1156 byte [] fam3 = Bytes.toBytes("fam3");
1157 byte [] fam4 = Bytes.toBytes("fam4");
1158
1159
1160 String method = this.getName();
1161 this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
1162 try {
1163 List<KeyValue> kvs = new ArrayList<KeyValue>();
1164 kvs.add(new KeyValue(row1, fam4, null, null));
1165
1166
1167
1168 byte [] family = fam2;
1169 try {
1170 Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
1171 deleteMap.put(family, kvs);
1172 region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
1173 } catch (Exception e) {
1174 assertTrue("Family " +new String(family)+ " does not exist", false);
1175 }
1176
1177
1178 boolean ok = false;
1179 family = fam4;
1180 try {
1181 Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
1182 deleteMap.put(family, kvs);
1183 region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
1184 } catch (Exception e) {
1185 ok = true;
1186 }
1187 assertEquals("Family " +new String(family)+ " does exist", true, ok);
1188 } finally {
1189 HRegion.closeHRegion(this.region);
1190 this.region = null;
1191 }
1192 }
1193
1194 public void testDelete_mixed() throws IOException, InterruptedException {
1195 byte [] tableName = Bytes.toBytes("testtable");
1196 byte [] fam = Bytes.toBytes("info");
1197 byte [][] families = {fam};
1198 String method = this.getName();
1199 this.region = initHRegion(tableName, method, conf, families);
1200 try {
1201 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1202
1203 byte [] row = Bytes.toBytes("table_name");
1204
1205 byte [] serverinfo = Bytes.toBytes("serverinfo");
1206 byte [] splitA = Bytes.toBytes("splitA");
1207 byte [] splitB = Bytes.toBytes("splitB");
1208
1209
1210 Put put = new Put(row);
1211 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1212 region.put(put);
1213
1214 put = new Put(row);
1215 put.add(fam, splitB, Bytes.toBytes("reference_B"));
1216 region.put(put);
1217
1218 put = new Put(row);
1219 put.add(fam, serverinfo, Bytes.toBytes("ip_address"));
1220 region.put(put);
1221
1222
1223 Delete delete = new Delete(row);
1224 delete.deleteColumns(fam, splitA);
1225 region.delete(delete, null, true);
1226
1227
1228 Get get = new Get(row).addColumn(fam, serverinfo);
1229 Result result = region.get(get, null);
1230 assertEquals(1, result.size());
1231
1232 get = new Get(row).addColumn(fam, splitA);
1233 result = region.get(get, null);
1234 assertEquals(0, result.size());
1235
1236 get = new Get(row).addColumn(fam, splitB);
1237 result = region.get(get, null);
1238 assertEquals(1, result.size());
1239
1240
1241 put = new Put(row);
1242 put.add(fam, splitA, Bytes.toBytes("reference_A"));
1243 region.put(put);
1244 get = new Get(row);
1245 result = region.get(get, null);
1246 assertEquals(3, result.size());
1247
1248
1249 delete = new Delete(row);
1250 region.delete(delete, null, false);
1251 assertEquals(0, region.get(get, null).size());
1252
1253 region.put(new Put(row).add(fam, splitA, Bytes.toBytes("reference_A")));
1254 result = region.get(get, null);
1255 assertEquals(1, result.size());
1256 } finally {
1257 HRegion.closeHRegion(this.region);
1258 this.region = null;
1259 }
1260 }
1261
1262 public void testDeleteRowWithFutureTs() throws IOException {
1263 byte [] tableName = Bytes.toBytes("testtable");
1264 byte [] fam = Bytes.toBytes("info");
1265 byte [][] families = {fam};
1266 String method = this.getName();
1267 this.region = initHRegion(tableName, method, conf, families);
1268 try {
1269 byte [] row = Bytes.toBytes("table_name");
1270
1271 byte [] serverinfo = Bytes.toBytes("serverinfo");
1272
1273
1274 Put put = new Put(row);
1275 put.add(fam, serverinfo, HConstants.LATEST_TIMESTAMP-5,Bytes.toBytes("value"));
1276 region.put(put);
1277
1278
1279 Delete delete = new Delete(row);
1280 region.delete(delete, null, true);
1281
1282
1283 Get get = new Get(row).addColumn(fam, serverinfo);
1284 Result result = region.get(get, null);
1285 assertEquals(1, result.size());
1286
1287
1288 delete = new Delete(row,HConstants.LATEST_TIMESTAMP-3,null);
1289 region.delete(delete, null, true);
1290
1291
1292 get = new Get(row).addColumn(fam, serverinfo);
1293 result = region.get(get, null);
1294 assertEquals(0, result.size());
1295 } finally {
1296 HRegion.closeHRegion(this.region);
1297 this.region = null;
1298 }
1299 }
1300
1301
1302
1303
1304
1305 public void testPutWithLatestTS() throws IOException {
1306 byte [] tableName = Bytes.toBytes("testtable");
1307 byte [] fam = Bytes.toBytes("info");
1308 byte [][] families = {fam};
1309 String method = this.getName();
1310 this.region = initHRegion(tableName, method, conf, families);
1311 try {
1312 byte [] row = Bytes.toBytes("row1");
1313
1314 byte [] qual = Bytes.toBytes("qual");
1315
1316
1317 Put put = new Put(row);
1318 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1319 region.put(put, false);
1320
1321
1322 Get get = new Get(row).addColumn(fam, qual);
1323 Result result = region.get(get, null);
1324 assertEquals(1, result.size());
1325 KeyValue kv = result.raw()[0];
1326 LOG.info("Got: " + kv);
1327 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1328 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1329
1330
1331
1332 row = Bytes.toBytes("row2");
1333 put = new Put(row);
1334 put.add(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
1335 region.put(put, true);
1336
1337
1338 get = new Get(row).addColumn(fam, qual);
1339 result = region.get(get, null);
1340 assertEquals(1, result.size());
1341 kv = result.raw()[0];
1342 LOG.info("Got: " + kv);
1343 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
1344 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
1345 } finally {
1346 HRegion.closeHRegion(this.region);
1347 this.region = null;
1348 }
1349
1350 }
1351
1352
1353
1354
1355
1356
1357
1358 public void testPutWithTsSlop() throws IOException {
1359 byte[] tableName = Bytes.toBytes("testtable");
1360 byte[] fam = Bytes.toBytes("info");
1361 byte[][] families = { fam };
1362 String method = this.getName();
1363 Configuration conf = HBaseConfiguration.create(this.conf);
1364
1365
1366 conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1367 this.region = initHRegion(tableName, method, conf, families);
1368 boolean caughtExcep = false;
1369 try {
1370 try {
1371
1372 region.put(new Put(row).add(fam, Bytes.toBytes("qual"), Bytes
1373 .toBytes("value")), false);
1374
1375 region.put(new Put(row).add(fam, Bytes.toBytes("qual"),
1376 System.currentTimeMillis() + 2000,
1377 Bytes.toBytes("value")), false);
1378 fail("Expected IOE for TS out of configured timerange");
1379 } catch (DoNotRetryIOException ioe) {
1380 LOG.debug("Received expected exception", ioe);
1381 caughtExcep = true;
1382 }
1383 assertTrue("Should catch FailedSanityCheckException", caughtExcep);
1384 } finally {
1385 HRegion.closeHRegion(this.region);
1386 this.region = null;
1387 }
1388 }
1389
1390 public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
1391 byte [] tableName = Bytes.toBytes("test_table");
1392 byte [] fam1 = Bytes.toBytes("columnA");
1393 byte [] fam2 = Bytes.toBytes("columnB");
1394 this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
1395 try {
1396 byte [] rowA = Bytes.toBytes("rowA");
1397 byte [] rowB = Bytes.toBytes("rowB");
1398
1399 byte [] value = Bytes.toBytes("value");
1400
1401 Delete delete = new Delete(rowA);
1402 delete.deleteFamily(fam1);
1403
1404 region.delete(delete, null, true);
1405
1406
1407 Put put = new Put(rowA);
1408 put.add(fam2, null, value);
1409 region.put(put);
1410
1411 put = new Put(rowB);
1412 put.add(fam1, null, value);
1413 put.add(fam2, null, value);
1414 region.put(put);
1415
1416 Scan scan = new Scan();
1417 scan.addFamily(fam1).addFamily(fam2);
1418 InternalScanner s = region.getScanner(scan);
1419 List<KeyValue> results = new ArrayList<KeyValue>();
1420 s.next(results);
1421 assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
1422
1423 results.clear();
1424 s.next(results);
1425 assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
1426 } finally {
1427 HRegion.closeHRegion(this.region);
1428 this.region = null;
1429 }
1430 }
1431
1432 public void testDeleteColumns_PostInsert() throws IOException,
1433 InterruptedException {
1434 Delete delete = new Delete(row);
1435 delete.deleteColumns(fam1, qual1);
1436 doTestDelete_AndPostInsert(delete);
1437 }
1438
1439 public void testDeleteFamily_PostInsert() throws IOException, InterruptedException {
1440 Delete delete = new Delete(row);
1441 delete.deleteFamily(fam1);
1442 doTestDelete_AndPostInsert(delete);
1443 }
1444
1445 public void doTestDelete_AndPostInsert(Delete delete)
1446 throws IOException, InterruptedException {
1447 this.region = initHRegion(tableName, getName(), conf, fam1);
1448 try {
1449 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
1450 Put put = new Put(row);
1451 put.add(fam1, qual1, value1);
1452 region.put(put);
1453
1454
1455 region.delete(delete, null, true);
1456
1457
1458
1459 put = new Put(row);
1460 put.add(fam1, qual1, value2);
1461 region.put(put);
1462
1463
1464 Get get = new Get(row);
1465 get.addColumn(fam1, qual1);
1466
1467 Result r = region.get(get, null);
1468 assertEquals(1, r.size());
1469 assertByteEquals(value2, r.getValue(fam1, qual1));
1470
1471
1472 Scan scan = new Scan(row);
1473 scan.addColumn(fam1, qual1);
1474 InternalScanner s = region.getScanner(scan);
1475
1476 List<KeyValue> results = new ArrayList<KeyValue>();
1477 assertEquals(false, s.next(results));
1478 assertEquals(1, results.size());
1479 KeyValue kv = results.get(0);
1480
1481 assertByteEquals(value2, kv.getValue());
1482 assertByteEquals(fam1, kv.getFamily());
1483 assertByteEquals(qual1, kv.getQualifier());
1484 assertByteEquals(row, kv.getRow());
1485 } finally {
1486 HRegion.closeHRegion(this.region);
1487 this.region = null;
1488 }
1489 }
1490
1491 public void testDelete_CheckTimestampUpdated()
1492 throws IOException {
1493 byte [] row1 = Bytes.toBytes("row1");
1494 byte [] col1 = Bytes.toBytes("col1");
1495 byte [] col2 = Bytes.toBytes("col2");
1496 byte [] col3 = Bytes.toBytes("col3");
1497
1498
1499 String method = this.getName();
1500 this.region = initHRegion(tableName, method, conf, fam1);
1501 try {
1502
1503 List<KeyValue> kvs = new ArrayList<KeyValue>();
1504 kvs.add(new KeyValue(row1, fam1, col1, null));
1505 kvs.add(new KeyValue(row1, fam1, col2, null));
1506 kvs.add(new KeyValue(row1, fam1, col3, null));
1507
1508 Map<byte[], List<KeyValue>> deleteMap = new HashMap<byte[], List<KeyValue>>();
1509 deleteMap.put(fam1, kvs);
1510 region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true);
1511
1512
1513
1514 long now = System.currentTimeMillis();
1515 KeyValue firstKv = region.getStore(fam1).memstore.kvset.first();
1516 assertTrue(firstKv.getTimestamp() <= now);
1517 now = firstKv.getTimestamp();
1518 for (KeyValue kv: region.getStore(fam1).memstore.kvset) {
1519 assertTrue(kv.getTimestamp() <= now);
1520 now = kv.getTimestamp();
1521 }
1522 } finally {
1523 HRegion.closeHRegion(this.region);
1524 this.region = null;
1525 }
1526 }
1527
1528
1529
1530
1531 public void testGet_FamilyChecker() throws IOException {
1532 byte [] tableName = Bytes.toBytes("testtable");
1533 byte [] row1 = Bytes.toBytes("row1");
1534 byte [] fam1 = Bytes.toBytes("fam1");
1535 byte [] fam2 = Bytes.toBytes("False");
1536 byte [] col1 = Bytes.toBytes("col1");
1537
1538
1539 String method = this.getName();
1540 this.region = initHRegion(tableName, method, conf, fam1);
1541 try {
1542 Get get = new Get(row1);
1543 get.addColumn(fam2, col1);
1544
1545
1546 try {
1547 region.get(get, null);
1548 } catch (DoNotRetryIOException e) {
1549 assertFalse(false);
1550 return;
1551 }
1552 assertFalse(true);
1553 } finally {
1554 HRegion.closeHRegion(this.region);
1555 this.region = null;
1556 }
1557 }
1558
1559 public void testGet_Basic() throws IOException {
1560 byte [] tableName = Bytes.toBytes("testtable");
1561 byte [] row1 = Bytes.toBytes("row1");
1562 byte [] fam1 = Bytes.toBytes("fam1");
1563 byte [] col1 = Bytes.toBytes("col1");
1564 byte [] col2 = Bytes.toBytes("col2");
1565 byte [] col3 = Bytes.toBytes("col3");
1566 byte [] col4 = Bytes.toBytes("col4");
1567 byte [] col5 = Bytes.toBytes("col5");
1568
1569
1570 String method = this.getName();
1571 this.region = initHRegion(tableName, method, conf, fam1);
1572 try {
1573
1574 Put put = new Put(row1);
1575 put.add(fam1, col1, null);
1576 put.add(fam1, col2, null);
1577 put.add(fam1, col3, null);
1578 put.add(fam1, col4, null);
1579 put.add(fam1, col5, null);
1580 region.put(put);
1581
1582 Get get = new Get(row1);
1583 get.addColumn(fam1, col2);
1584 get.addColumn(fam1, col4);
1585
1586 KeyValue kv1 = new KeyValue(row1, fam1, col2);
1587 KeyValue kv2 = new KeyValue(row1, fam1, col4);
1588 KeyValue [] expected = {kv1, kv2};
1589
1590
1591 Result res = region.get(get, null);
1592 assertEquals(expected.length, res.size());
1593 for(int i=0; i<res.size(); i++){
1594 assertEquals(0,
1595 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1596 assertEquals(0,
1597 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1598 assertEquals(0,
1599 Bytes.compareTo(
1600 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1601 }
1602
1603
1604 Get g = new Get(row1);
1605 final int count = 2;
1606 g.setFilter(new ColumnCountGetFilter(count));
1607 res = region.get(g, null);
1608 assertEquals(count, res.size());
1609 } finally {
1610 HRegion.closeHRegion(this.region);
1611 this.region = null;
1612 }
1613 }
1614
1615 public void testGet_Empty() throws IOException {
1616 byte [] tableName = Bytes.toBytes("emptytable");
1617 byte [] row = Bytes.toBytes("row");
1618 byte [] fam = Bytes.toBytes("fam");
1619
1620 String method = this.getName();
1621 this.region = initHRegion(tableName, method, conf, fam);
1622 try {
1623 Get get = new Get(row);
1624 get.addFamily(fam);
1625 Result r = region.get(get, null);
1626
1627 assertTrue(r.isEmpty());
1628 } finally {
1629 HRegion.closeHRegion(this.region);
1630 this.region = null;
1631 }
1632 }
1633
1634
1635
1636
1637
1638 public void stestGet_Root() throws IOException {
1639
1640 String method = this.getName();
1641 this.region = initHRegion(HConstants.ROOT_TABLE_NAME,
1642 method, conf, HConstants.CATALOG_FAMILY);
1643 try {
1644
1645 Put put = new Put(HConstants.EMPTY_START_ROW);
1646 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, null);
1647 region.put(put);
1648
1649 Get get = new Get(HConstants.EMPTY_START_ROW);
1650 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1651
1652
1653 KeyValue kv1 = new KeyValue(HConstants.EMPTY_START_ROW,
1654 HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1655 KeyValue [] expected = {kv1};
1656
1657
1658 Result res = region.get(get, null);
1659
1660 assertEquals(expected.length, res.size());
1661 for(int i=0; i<res.size(); i++){
1662 assertEquals(0,
1663 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1664 assertEquals(0,
1665 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1666 assertEquals(0,
1667 Bytes.compareTo(
1668 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1669 }
1670
1671
1672 region.flushcache();
1673
1674
1675 res = region.get(get, null);
1676
1677 assertEquals(expected.length, res.size());
1678 for(int i=0; i<res.size(); i++){
1679 assertEquals(0,
1680 Bytes.compareTo(expected[i].getRow(), res.raw()[i].getRow()));
1681 assertEquals(0,
1682 Bytes.compareTo(expected[i].getFamily(), res.raw()[i].getFamily()));
1683 assertEquals(0,
1684 Bytes.compareTo(
1685 expected[i].getQualifier(), res.raw()[i].getQualifier()));
1686 }
1687
1688
1689 Scan scan = new Scan();
1690 scan.addFamily(HConstants.CATALOG_FAMILY);
1691 InternalScanner s = region.getScanner(scan);
1692 List<KeyValue> result = new ArrayList<KeyValue>();
1693 s.next(result);
1694
1695 assertEquals(expected.length, result.size());
1696 for(int i=0; i<res.size(); i++){
1697 assertEquals(0,
1698 Bytes.compareTo(expected[i].getRow(), result.get(i).getRow()));
1699 assertEquals(0,
1700 Bytes.compareTo(expected[i].getFamily(), result.get(i).getFamily()));
1701 assertEquals(0,
1702 Bytes.compareTo(
1703 expected[i].getQualifier(), result.get(i).getQualifier()));
1704 }
1705 } finally {
1706 HRegion.closeHRegion(this.region);
1707 this.region = null;
1708 }
1709 }
1710
1711
1712
1713
1714 public void testLocks() throws IOException{
1715 byte [] tableName = Bytes.toBytes("testtable");
1716 byte [][] families = {fam1, fam2, fam3};
1717
1718 Configuration hc = initSplit();
1719
1720 String method = this.getName();
1721 this.region = initHRegion(tableName, method, hc, families);
1722 try {
1723 final int threadCount = 10;
1724 final int lockCount = 10;
1725
1726 List<Thread>threads = new ArrayList<Thread>(threadCount);
1727 for (int i = 0; i < threadCount; i++) {
1728 threads.add(new Thread(Integer.toString(i)) {
1729 @Override
1730 public void run() {
1731 Integer [] lockids = new Integer[lockCount];
1732
1733 for (int i = 0; i < lockCount; i++) {
1734 try {
1735 byte [] rowid = Bytes.toBytes(Integer.toString(i));
1736 lockids[i] = region.obtainRowLock(rowid);
1737 assertEquals(rowid, region.getRowFromLock(lockids[i]));
1738 LOG.debug(getName() + " locked " + Bytes.toString(rowid));
1739 } catch (IOException e) {
1740 e.printStackTrace();
1741 }
1742 }
1743 LOG.debug(getName() + " set " +
1744 Integer.toString(lockCount) + " locks");
1745
1746
1747 for (int i = lockCount - 1; i >= 0; i--) {
1748 region.releaseRowLock(lockids[i]);
1749 LOG.debug(getName() + " unlocked " + i);
1750 }
1751 LOG.debug(getName() + " released " +
1752 Integer.toString(lockCount) + " locks");
1753 }
1754 });
1755 }
1756
1757
1758 for (Thread t : threads) {
1759 t.start();
1760 }
1761
1762
1763 for (Thread t: threads) {
1764 while (t.isAlive()) {
1765 try {
1766 Thread.sleep(1);
1767 } catch (InterruptedException e) {
1768
1769 }
1770 }
1771 }
1772 LOG.info("locks completed.");
1773 } finally {
1774 HRegion.closeHRegion(this.region);
1775 this.region = null;
1776 }
1777 }
1778
1779
1780
1781
1782 public void testMerge() throws IOException {
1783 byte [] tableName = Bytes.toBytes("testtable");
1784 byte [][] families = {fam1, fam2, fam3};
1785 Configuration hc = initSplit();
1786
1787 String method = this.getName();
1788 this.region = initHRegion(tableName, method, hc, families);
1789 try {
1790 LOG.info("" + addContent(region, fam3));
1791 region.flushcache();
1792 region.compactStores();
1793 byte [] splitRow = region.checkSplit();
1794 assertNotNull(splitRow);
1795 LOG.info("SplitRow: " + Bytes.toString(splitRow));
1796 HRegion [] subregions = splitRegion(region, splitRow);
1797 try {
1798
1799 for (int i = 0; i < subregions.length; i++) {
1800 openClosedRegion(subregions[i]);
1801 subregions[i].compactStores();
1802 }
1803 Path oldRegionPath = region.getRegionDir();
1804 Path oldRegion1 = subregions[0].getRegionDir();
1805 Path oldRegion2 = subregions[1].getRegionDir();
1806 long startTime = System.currentTimeMillis();
1807 region = HRegion.mergeAdjacent(subregions[0], subregions[1]);
1808 LOG.info("Merge regions elapsed time: " +
1809 ((System.currentTimeMillis() - startTime) / 1000.0));
1810 fs.delete(oldRegion1, true);
1811 fs.delete(oldRegion2, true);
1812 fs.delete(oldRegionPath, true);
1813 LOG.info("splitAndMerge completed.");
1814 } finally {
1815 for (int i = 0; i < subregions.length; i++) {
1816 try {
1817 subregions[i].close();
1818 subregions[i].getLog().closeAndDelete();
1819 } catch (IOException e) {
1820
1821 }
1822 }
1823 }
1824 } finally {
1825 HRegion.closeHRegion(this.region);
1826 this.region = null;
1827 }
1828 }
1829
1830
1831
1832
1833
1834
1835
1836 HRegion [] splitRegion(final HRegion parent, final byte [] midkey)
1837 throws IOException {
1838 PairOfSameType<HRegion> result = null;
1839 SplitTransaction st = new SplitTransaction(parent, midkey);
1840
1841
1842 if (!st.prepare()) return null;
1843 try {
1844 result = st.execute(null, null);
1845 } catch (IOException ioe) {
1846 try {
1847 LOG.info("Running rollback of failed split of " +
1848 parent.getRegionNameAsString() + "; " + ioe.getMessage());
1849 st.rollback(null, null);
1850 LOG.info("Successful rollback of failed split of " +
1851 parent.getRegionNameAsString());
1852 return null;
1853 } catch (RuntimeException e) {
1854
1855 LOG.info("Failed rollback of failed split of " +
1856 parent.getRegionNameAsString() + " -- aborting server", e);
1857 }
1858 }
1859 return new HRegion [] {result.getFirst(), result.getSecond()};
1860 }
1861
1862
1863
1864
1865 public void testGetScanner_WithOkFamilies() throws IOException {
1866 byte [] tableName = Bytes.toBytes("testtable");
1867 byte [] fam1 = Bytes.toBytes("fam1");
1868 byte [] fam2 = Bytes.toBytes("fam2");
1869
1870 byte [][] families = {fam1, fam2};
1871
1872
1873 String method = this.getName();
1874 this.region = initHRegion(tableName, method, conf, families);
1875 try {
1876 Scan scan = new Scan();
1877 scan.addFamily(fam1);
1878 scan.addFamily(fam2);
1879 try {
1880 region.getScanner(scan);
1881 } catch (Exception e) {
1882 assertTrue("Families could not be found in Region", false);
1883 }
1884 } finally {
1885 HRegion.closeHRegion(this.region);
1886 this.region = null;
1887 }
1888 }
1889
1890 public void testGetScanner_WithNotOkFamilies() throws IOException {
1891 byte [] tableName = Bytes.toBytes("testtable");
1892 byte [] fam1 = Bytes.toBytes("fam1");
1893 byte [] fam2 = Bytes.toBytes("fam2");
1894
1895 byte [][] families = {fam1};
1896
1897
1898 String method = this.getName();
1899 this.region = initHRegion(tableName, method, conf, families);
1900 try {
1901 Scan scan = new Scan();
1902 scan.addFamily(fam2);
1903 boolean ok = false;
1904 try {
1905 region.getScanner(scan);
1906 } catch (Exception e) {
1907 ok = true;
1908 }
1909 assertTrue("Families could not be found in Region", ok);
1910 } finally {
1911 HRegion.closeHRegion(this.region);
1912 this.region = null;
1913 }
1914 }
1915
1916 public void testGetScanner_WithNoFamilies() throws IOException {
1917 byte [] tableName = Bytes.toBytes("testtable");
1918 byte [] row1 = Bytes.toBytes("row1");
1919 byte [] fam1 = Bytes.toBytes("fam1");
1920 byte [] fam2 = Bytes.toBytes("fam2");
1921 byte [] fam3 = Bytes.toBytes("fam3");
1922 byte [] fam4 = Bytes.toBytes("fam4");
1923
1924 byte [][] families = {fam1, fam2, fam3, fam4};
1925
1926
1927 String method = this.getName();
1928 this.region = initHRegion(tableName, method, conf, families);
1929 try {
1930
1931
1932 Put put = new Put(row1);
1933 put.add(fam1, null, null);
1934 put.add(fam2, null, null);
1935 put.add(fam3, null, null);
1936 put.add(fam4, null, null);
1937 region.put(put);
1938
1939 Scan scan = null;
1940 HRegion.RegionScannerImpl is = null;
1941
1942
1943
1944 scan = new Scan();
1945 scan.addFamily(fam2);
1946 scan.addFamily(fam4);
1947 is = (RegionScannerImpl) region.getScanner(scan);
1948 MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1949 assertEquals(1, ((RegionScannerImpl)is).storeHeap.getHeap().size());
1950
1951 scan = new Scan();
1952 is = (RegionScannerImpl) region.getScanner(scan);
1953 MultiVersionConsistencyControl.resetThreadReadPoint(region.getMVCC());
1954 assertEquals(families.length -1,
1955 ((RegionScannerImpl)is).storeHeap.getHeap().size());
1956 } finally {
1957 HRegion.closeHRegion(this.region);
1958 this.region = null;
1959 }
1960 }
1961
1962
1963
1964
1965
1966 public void testGetScanner_WithRegionClosed() throws IOException {
1967 byte[] tableName = Bytes.toBytes("testtable");
1968 byte[] fam1 = Bytes.toBytes("fam1");
1969 byte[] fam2 = Bytes.toBytes("fam2");
1970
1971 byte[][] families = {fam1, fam2};
1972
1973
1974 String method = this.getName();
1975 try {
1976 this.region = initHRegion(tableName, method, conf, families);
1977 } catch (IOException e) {
1978 e.printStackTrace();
1979 fail("Got IOException during initHRegion, " + e.getMessage());
1980 }
1981 try {
1982 region.closed.set(true);
1983 try {
1984 region.getScanner(null);
1985 fail("Expected to get an exception during getScanner on a region that is closed");
1986 } catch (org.apache.hadoop.hbase.NotServingRegionException e) {
1987
1988 } catch (IOException e) {
1989 fail("Got wrong type of exception - should be a NotServingRegionException, but was an IOException: "
1990 + e.getMessage());
1991 }
1992 } finally {
1993 HRegion.closeHRegion(this.region);
1994 this.region = null;
1995 }
1996 }
1997
1998 public void testRegionScanner_Next() throws IOException {
1999 byte [] tableName = Bytes.toBytes("testtable");
2000 byte [] row1 = Bytes.toBytes("row1");
2001 byte [] row2 = Bytes.toBytes("row2");
2002 byte [] fam1 = Bytes.toBytes("fam1");
2003 byte [] fam2 = Bytes.toBytes("fam2");
2004 byte [] fam3 = Bytes.toBytes("fam3");
2005 byte [] fam4 = Bytes.toBytes("fam4");
2006
2007 byte [][] families = {fam1, fam2, fam3, fam4};
2008 long ts = System.currentTimeMillis();
2009
2010
2011 String method = this.getName();
2012 this.region = initHRegion(tableName, method, conf, families);
2013 try {
2014
2015 Put put = null;
2016 put = new Put(row1);
2017 put.add(fam1, null, ts, null);
2018 put.add(fam2, null, ts, null);
2019 put.add(fam3, null, ts, null);
2020 put.add(fam4, null, ts, null);
2021 region.put(put);
2022
2023 put = new Put(row2);
2024 put.add(fam1, null, ts, null);
2025 put.add(fam2, null, ts, null);
2026 put.add(fam3, null, ts, null);
2027 put.add(fam4, null, ts, null);
2028 region.put(put);
2029
2030 Scan scan = new Scan();
2031 scan.addFamily(fam2);
2032 scan.addFamily(fam4);
2033 InternalScanner is = region.getScanner(scan);
2034
2035 List<KeyValue> res = null;
2036
2037
2038 List<KeyValue> expected1 = new ArrayList<KeyValue>();
2039 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
2040 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
2041
2042 res = new ArrayList<KeyValue>();
2043 is.next(res);
2044 for(int i=0; i<res.size(); i++) {
2045 assertEquals(expected1.get(i), res.get(i));
2046 }
2047
2048
2049 List<KeyValue> expected2 = new ArrayList<KeyValue>();
2050 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
2051 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
2052
2053 res = new ArrayList<KeyValue>();
2054 is.next(res);
2055 for(int i=0; i<res.size(); i++) {
2056 assertEquals(expected2.get(i), res.get(i));
2057 }
2058 } finally {
2059 HRegion.closeHRegion(this.region);
2060 this.region = null;
2061 }
2062 }
2063
2064 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions()
2065 throws IOException {
2066 byte [] tableName = Bytes.toBytes("testtable");
2067 byte [] row1 = Bytes.toBytes("row1");
2068 byte [] qf1 = Bytes.toBytes("qualifier1");
2069 byte [] qf2 = Bytes.toBytes("qualifier2");
2070 byte [] fam1 = Bytes.toBytes("fam1");
2071 byte [][] families = {fam1};
2072
2073 long ts1 = System.currentTimeMillis();
2074 long ts2 = ts1 + 1;
2075 long ts3 = ts1 + 2;
2076
2077
2078 String method = this.getName();
2079 this.region = initHRegion(tableName, method, conf, families);
2080 try {
2081
2082 Put put = null;
2083 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2084 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2085 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2086
2087 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2088 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2089 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2090
2091 put = new Put(row1);
2092 put.add(kv13);
2093 put.add(kv12);
2094 put.add(kv11);
2095 put.add(kv23);
2096 put.add(kv22);
2097 put.add(kv21);
2098 region.put(put);
2099
2100
2101 List<KeyValue> expected = new ArrayList<KeyValue>();
2102 expected.add(kv13);
2103 expected.add(kv12);
2104
2105 Scan scan = new Scan(row1);
2106 scan.addColumn(fam1, qf1);
2107 scan.setMaxVersions(MAX_VERSIONS);
2108 List<KeyValue> actual = new ArrayList<KeyValue>();
2109 InternalScanner scanner = region.getScanner(scan);
2110
2111 boolean hasNext = scanner.next(actual);
2112 assertEquals(false, hasNext);
2113
2114
2115 for(int i=0; i<expected.size(); i++) {
2116 assertEquals(expected.get(i), actual.get(i));
2117 }
2118 } finally {
2119 HRegion.closeHRegion(this.region);
2120 this.region = null;
2121 }
2122 }
2123
2124 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions()
2125 throws IOException{
2126 byte [] tableName = Bytes.toBytes("testtable");
2127 byte [] row1 = Bytes.toBytes("row1");
2128 byte [] qf1 = Bytes.toBytes("qualifier1");
2129 byte [] qf2 = Bytes.toBytes("qualifier2");
2130 byte [] fam1 = Bytes.toBytes("fam1");
2131 byte [][] families = {fam1};
2132
2133 long ts1 = 1;
2134 long ts2 = ts1 + 1;
2135 long ts3 = ts1 + 2;
2136
2137
2138 String method = this.getName();
2139 this.region = initHRegion(tableName, method, conf, families);
2140 try {
2141
2142 Put put = null;
2143 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2144 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2145 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2146
2147 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2148 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2149 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2150
2151 put = new Put(row1);
2152 put.add(kv13);
2153 put.add(kv12);
2154 put.add(kv11);
2155 put.add(kv23);
2156 put.add(kv22);
2157 put.add(kv21);
2158 region.put(put);
2159 region.flushcache();
2160
2161
2162 List<KeyValue> expected = new ArrayList<KeyValue>();
2163 expected.add(kv13);
2164 expected.add(kv12);
2165 expected.add(kv23);
2166 expected.add(kv22);
2167
2168 Scan scan = new Scan(row1);
2169 scan.addColumn(fam1, qf1);
2170 scan.addColumn(fam1, qf2);
2171 scan.setMaxVersions(MAX_VERSIONS);
2172 List<KeyValue> actual = new ArrayList<KeyValue>();
2173 InternalScanner scanner = region.getScanner(scan);
2174
2175 boolean hasNext = scanner.next(actual);
2176 assertEquals(false, hasNext);
2177
2178
2179 for(int i=0; i<expected.size(); i++) {
2180 assertEquals(expected.get(i), actual.get(i));
2181 }
2182 } finally {
2183 HRegion.closeHRegion(this.region);
2184 this.region = null;
2185 }
2186 }
2187
2188 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
2189 throws IOException {
2190 byte [] tableName = Bytes.toBytes("testtable");
2191 byte [] row1 = Bytes.toBytes("row1");
2192 byte [] fam1 = Bytes.toBytes("fam1");
2193 byte [][] families = {fam1};
2194 byte [] qf1 = Bytes.toBytes("qualifier1");
2195 byte [] qf2 = Bytes.toBytes("qualifier2");
2196
2197 long ts1 = 1;
2198 long ts2 = ts1 + 1;
2199 long ts3 = ts1 + 2;
2200 long ts4 = ts1 + 3;
2201
2202
2203 String method = this.getName();
2204 this.region = initHRegion(tableName, method, conf, families);
2205 try {
2206
2207 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2208 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2209 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2210 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2211
2212 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2213 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2214 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2215 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2216
2217 Put put = null;
2218 put = new Put(row1);
2219 put.add(kv14);
2220 put.add(kv24);
2221 region.put(put);
2222 region.flushcache();
2223
2224 put = new Put(row1);
2225 put.add(kv23);
2226 put.add(kv13);
2227 region.put(put);
2228 region.flushcache();
2229
2230 put = new Put(row1);
2231 put.add(kv22);
2232 put.add(kv12);
2233 region.put(put);
2234 region.flushcache();
2235
2236 put = new Put(row1);
2237 put.add(kv21);
2238 put.add(kv11);
2239 region.put(put);
2240
2241
2242 List<KeyValue> expected = new ArrayList<KeyValue>();
2243 expected.add(kv14);
2244 expected.add(kv13);
2245 expected.add(kv12);
2246 expected.add(kv24);
2247 expected.add(kv23);
2248 expected.add(kv22);
2249
2250 Scan scan = new Scan(row1);
2251 scan.addColumn(fam1, qf1);
2252 scan.addColumn(fam1, qf2);
2253 int versions = 3;
2254 scan.setMaxVersions(versions);
2255 List<KeyValue> actual = new ArrayList<KeyValue>();
2256 InternalScanner scanner = region.getScanner(scan);
2257
2258 boolean hasNext = scanner.next(actual);
2259 assertEquals(false, hasNext);
2260
2261
2262 for(int i=0; i<expected.size(); i++) {
2263 assertEquals(expected.get(i), actual.get(i));
2264 }
2265 } finally {
2266 HRegion.closeHRegion(this.region);
2267 this.region = null;
2268 }
2269 }
2270
2271 public void testScanner_Wildcard_FromMemStore_EnforceVersions()
2272 throws IOException {
2273 byte [] tableName = Bytes.toBytes("testtable");
2274 byte [] row1 = Bytes.toBytes("row1");
2275 byte [] qf1 = Bytes.toBytes("qualifier1");
2276 byte [] qf2 = Bytes.toBytes("qualifier2");
2277 byte [] fam1 = Bytes.toBytes("fam1");
2278 byte [][] families = {fam1};
2279
2280 long ts1 = System.currentTimeMillis();
2281 long ts2 = ts1 + 1;
2282 long ts3 = ts1 + 2;
2283
2284
2285 String method = this.getName();
2286 this.region = initHRegion(tableName, method, conf, families);
2287 try {
2288
2289 Put put = null;
2290 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2291 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2292 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2293
2294 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2295 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2296 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2297
2298 put = new Put(row1);
2299 put.add(kv13);
2300 put.add(kv12);
2301 put.add(kv11);
2302 put.add(kv23);
2303 put.add(kv22);
2304 put.add(kv21);
2305 region.put(put);
2306
2307
2308 List<KeyValue> expected = new ArrayList<KeyValue>();
2309 expected.add(kv13);
2310 expected.add(kv12);
2311 expected.add(kv23);
2312 expected.add(kv22);
2313
2314 Scan scan = new Scan(row1);
2315 scan.addFamily(fam1);
2316 scan.setMaxVersions(MAX_VERSIONS);
2317 List<KeyValue> actual = new ArrayList<KeyValue>();
2318 InternalScanner scanner = region.getScanner(scan);
2319
2320 boolean hasNext = scanner.next(actual);
2321 assertEquals(false, hasNext);
2322
2323
2324 for(int i=0; i<expected.size(); i++) {
2325 assertEquals(expected.get(i), actual.get(i));
2326 }
2327 } finally {
2328 HRegion.closeHRegion(this.region);
2329 this.region = null;
2330 }
2331 }
2332
2333 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions()
2334 throws IOException{
2335 byte [] tableName = Bytes.toBytes("testtable");
2336 byte [] row1 = Bytes.toBytes("row1");
2337 byte [] qf1 = Bytes.toBytes("qualifier1");
2338 byte [] qf2 = Bytes.toBytes("qualifier2");
2339 byte [] fam1 = Bytes.toBytes("fam1");
2340
2341 long ts1 = 1;
2342 long ts2 = ts1 + 1;
2343 long ts3 = ts1 + 2;
2344
2345
2346 String method = this.getName();
2347 this.region = initHRegion(tableName, method, conf, fam1);
2348 try {
2349
2350 Put put = null;
2351 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2352 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2353 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2354
2355 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2356 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2357 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2358
2359 put = new Put(row1);
2360 put.add(kv13);
2361 put.add(kv12);
2362 put.add(kv11);
2363 put.add(kv23);
2364 put.add(kv22);
2365 put.add(kv21);
2366 region.put(put);
2367 region.flushcache();
2368
2369
2370 List<KeyValue> expected = new ArrayList<KeyValue>();
2371 expected.add(kv13);
2372 expected.add(kv12);
2373 expected.add(kv23);
2374 expected.add(kv22);
2375
2376 Scan scan = new Scan(row1);
2377 scan.addFamily(fam1);
2378 scan.setMaxVersions(MAX_VERSIONS);
2379 List<KeyValue> actual = new ArrayList<KeyValue>();
2380 InternalScanner scanner = region.getScanner(scan);
2381
2382 boolean hasNext = scanner.next(actual);
2383 assertEquals(false, hasNext);
2384
2385
2386 for(int i=0; i<expected.size(); i++) {
2387 assertEquals(expected.get(i), actual.get(i));
2388 }
2389 } finally {
2390 HRegion.closeHRegion(this.region);
2391 this.region = null;
2392 }
2393 }
2394
2395 public void testScanner_StopRow1542() throws IOException {
2396 byte [] tableName = Bytes.toBytes("test_table");
2397 byte [] family = Bytes.toBytes("testFamily");
2398 this.region = initHRegion(tableName, getName(), conf, family);
2399 try {
2400 byte [] row1 = Bytes.toBytes("row111");
2401 byte [] row2 = Bytes.toBytes("row222");
2402 byte [] row3 = Bytes.toBytes("row333");
2403 byte [] row4 = Bytes.toBytes("row444");
2404 byte [] row5 = Bytes.toBytes("row555");
2405
2406 byte [] col1 = Bytes.toBytes("Pub111");
2407 byte [] col2 = Bytes.toBytes("Pub222");
2408
2409
2410 Put put = new Put(row1);
2411 put.add(family, col1, Bytes.toBytes(10L));
2412 region.put(put);
2413
2414 put = new Put(row2);
2415 put.add(family, col1, Bytes.toBytes(15L));
2416 region.put(put);
2417
2418 put = new Put(row3);
2419 put.add(family, col2, Bytes.toBytes(20L));
2420 region.put(put);
2421
2422 put = new Put(row4);
2423 put.add(family, col2, Bytes.toBytes(30L));
2424 region.put(put);
2425
2426 put = new Put(row5);
2427 put.add(family, col1, Bytes.toBytes(40L));
2428 region.put(put);
2429
2430 Scan scan = new Scan(row3, row4);
2431 scan.setMaxVersions();
2432 scan.addColumn(family, col1);
2433 InternalScanner s = region.getScanner(scan);
2434
2435 List<KeyValue> results = new ArrayList<KeyValue>();
2436 assertEquals(false, s.next(results));
2437 assertEquals(0, results.size());
2438 } finally {
2439 HRegion.closeHRegion(this.region);
2440 this.region = null;
2441 }
2442 }
2443
2444 public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
2445 this.region = initHRegion(tableName, getName(), conf, fam1);
2446 try {
2447 long value = 1L;
2448 long amount = 3L;
2449
2450 Put put = new Put(row);
2451 put.add(fam1, qual1, Bytes.toBytes(value));
2452 region.put(put);
2453
2454 long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
2455
2456 assertEquals(value+amount, result);
2457
2458 Store store = region.getStore(fam1);
2459
2460 assertEquals(1, store.memstore.kvset.size());
2461 assertTrue(store.memstore.snapshot.isEmpty());
2462
2463 assertICV(row, fam1, qual1, value+amount);
2464 } finally {
2465 HRegion.closeHRegion(this.region);
2466 this.region = null;
2467 }
2468 }
2469
2470 public void testIncrementColumnValue_BumpSnapshot() throws IOException {
2471 ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
2472 EnvironmentEdgeManagerTestHelper.injectEdge(mee);
2473 this.region = initHRegion(tableName, getName(), conf, fam1);
2474 try {
2475 long value = 42L;
2476 long incr = 44L;
2477
2478
2479 Put put = new Put(row);
2480 put.add(fam1, qual1, Bytes.toBytes(value));
2481 region.put(put);
2482
2483
2484 Store s = region.getStore(fam1);
2485 s.snapshot();
2486
2487
2488 long newVal = region.incrementColumnValue(row, fam1, qual1,
2489 incr, false);
2490
2491 assertEquals(value+incr, newVal);
2492
2493
2494 Get get = new Get(row);
2495 get.setMaxVersions();
2496 get.addColumn(fam1,qual1);
2497
2498 Result r = region.get(get, null);
2499 assertEquals(2, r.size());
2500 KeyValue first = r.raw()[0];
2501 KeyValue second = r.raw()[1];
2502
2503 assertTrue("ICV failed to upgrade timestamp",
2504 first.getTimestamp() != second.getTimestamp());
2505 } finally {
2506 HRegion.closeHRegion(this.region);
2507 this.region = null;
2508 }
2509 }
2510
2511 public void testIncrementColumnValue_ConcurrentFlush() throws IOException {
2512 this.region = initHRegion(tableName, getName(), conf, fam1);
2513 try {
2514 long value = 1L;
2515 long amount = 3L;
2516
2517 Put put = new Put(row);
2518 put.add(fam1, qual1, Bytes.toBytes(value));
2519 region.put(put);
2520
2521
2522 Thread t = new Thread() {
2523 public void run() {
2524 try {
2525 region.flushcache();
2526 } catch (IOException e) {
2527 LOG.info("test ICV, got IOE during flushcache()");
2528 }
2529 }
2530 };
2531 t.start();
2532 long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
2533 assertEquals(value+amount, r);
2534
2535
2536 assertICV(row, fam1, qual1, value+amount);
2537 } finally {
2538 HRegion.closeHRegion(this.region);
2539 this.region = null;
2540 }
2541 }
2542
2543 public void testIncrementColumnValue_heapSize() throws IOException {
2544 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
2545
2546 this.region = initHRegion(tableName, getName(), conf, fam1);
2547 try {
2548 long byAmount = 1L;
2549 long size;
2550
2551 for( int i = 0; i < 1000 ; i++) {
2552 region.incrementColumnValue(row, fam1, qual1, byAmount, true);
2553
2554 size = region.memstoreSize.get();
2555 assertTrue("memstore size: " + size, size >= 0);
2556 }
2557 } finally {
2558 HRegion.closeHRegion(this.region);
2559 this.region = null;
2560 }
2561 }
2562
2563 public void testIncrementColumnValue_UpdatingInPlace_Negative()
2564 throws IOException {
2565 this.region = initHRegion(tableName, getName(), conf, fam1);
2566 try {
2567 long value = 3L;
2568 long amount = -1L;
2569
2570 Put put = new Put(row);
2571 put.add(fam1, qual1, Bytes.toBytes(value));
2572 region.put(put);
2573
2574 long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
2575 assertEquals(value+amount, result);
2576
2577 assertICV(row, fam1, qual1, value+amount);
2578 } finally {
2579 HRegion.closeHRegion(this.region);
2580 this.region = null;
2581 }
2582 }
2583
2584 public void testIncrementColumnValue_AddingNew()
2585 throws IOException {
2586 this.region = initHRegion(tableName, getName(), conf, fam1);
2587 try {
2588 long value = 1L;
2589 long amount = 3L;
2590
2591 Put put = new Put(row);
2592 put.add(fam1, qual1, Bytes.toBytes(value));
2593 put.add(fam1, qual2, Bytes.toBytes(value));
2594 region.put(put);
2595
2596 long result = region.incrementColumnValue(row, fam1, qual3, amount, true);
2597 assertEquals(amount, result);
2598
2599 Get get = new Get(row);
2600 get.addColumn(fam1, qual3);
2601 Result rr = region.get(get, null);
2602 assertEquals(1, rr.size());
2603
2604
2605 assertICV(row, fam1, qual1, value);
2606 assertICV(row, fam1, qual2, value);
2607 assertICV(row, fam1, qual3, amount);
2608 } finally {
2609 HRegion.closeHRegion(this.region);
2610 this.region = null;
2611 }
2612 }
2613
2614 public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
2615 this.region = initHRegion(tableName, getName(), conf, fam1);
2616 try {
2617 long value = 1L;
2618 long amount = 3L;
2619
2620 Put put = new Put(row);
2621 put.add(fam1, qual1, Bytes.toBytes(value));
2622 put.add(fam1, qual2, Bytes.toBytes(value));
2623 region.put(put);
2624
2625
2626 region.flushcache();
2627
2628 Store store = region.getStore(fam1);
2629 assertEquals(0, store.memstore.kvset.size());
2630
2631 long r = region.incrementColumnValue(row, fam1, qual1, amount, true);
2632 assertEquals(value+amount, r);
2633
2634 assertICV(row, fam1, qual1, value+amount);
2635 } finally {
2636 HRegion.closeHRegion(this.region);
2637 this.region = null;
2638 }
2639 }
2640
2641 public void testIncrementColumnValue_AddingNewAfterSFCheck()
2642 throws IOException {
2643 this.region = initHRegion(tableName, getName(), conf, fam1);
2644 try {
2645 long value = 1L;
2646 long amount = 3L;
2647
2648 Put put = new Put(row);
2649 put.add(fam1, qual1, Bytes.toBytes(value));
2650 put.add(fam1, qual2, Bytes.toBytes(value));
2651 region.put(put);
2652 region.flushcache();
2653
2654 Store store = region.getStore(fam1);
2655 assertEquals(0, store.memstore.kvset.size());
2656
2657 long r = region.incrementColumnValue(row, fam1, qual3, amount, true);
2658 assertEquals(amount, r);
2659
2660 assertICV(row, fam1, qual3, amount);
2661
2662 region.flushcache();
2663
2664
2665 assertICV(row, fam1, qual3, amount);
2666 } finally {
2667 HRegion.closeHRegion(this.region);
2668 this.region = null;
2669 }
2670 }
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681 public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws IOException {
2682 this.region = initHRegion(tableName, getName(), conf, fam1);
2683 try {
2684 long value = 1L;
2685 long amount = 3L;
2686 long now = EnvironmentEdgeManager.currentTimeMillis();
2687 ManualEnvironmentEdge mock = new ManualEnvironmentEdge();
2688 mock.setValue(now);
2689 EnvironmentEdgeManagerTestHelper.injectEdge(mock);
2690
2691
2692 Put put = new Put(row);
2693 put.add(fam1, qual1, now, Bytes.toBytes(value));
2694 region.put(put);
2695
2696 long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
2697
2698 assertEquals(value+amount, result);
2699
2700 Store store = region.getStore(fam1);
2701
2702 assertEquals(1, store.memstore.kvset.size());
2703 assertTrue(store.memstore.snapshot.isEmpty());
2704
2705 assertICV(row, fam1, qual1, value+amount);
2706
2707
2708 put = new Put(row);
2709 put.add(fam1, qual2, now+1, Bytes.toBytes(value));
2710 region.put(put);
2711
2712 result = region.incrementColumnValue(row, fam1, qual2, amount, true);
2713
2714 assertEquals(value+amount, result);
2715
2716 store = region.getStore(fam1);
2717
2718 assertEquals(2, store.memstore.kvset.size());
2719 assertTrue(store.memstore.snapshot.isEmpty());
2720
2721 assertICV(row, fam1, qual2, value+amount);
2722 EnvironmentEdgeManagerTestHelper.reset();
2723 } finally {
2724 HRegion.closeHRegion(this.region);
2725 this.region = null;
2726 }
2727 }
2728
2729 public void testIncrementColumnValue_WrongInitialSize() throws IOException {
2730 this.region = initHRegion(tableName, getName(), conf, fam1);
2731 try {
2732 byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
2733 int row1Field1 = 0;
2734 int row1Field2 = 1;
2735 Put put1 = new Put(row1);
2736 put1.add(fam1, qual1, Bytes.toBytes(row1Field1));
2737 put1.add(fam1, qual2, Bytes.toBytes(row1Field2));
2738 region.put(put1);
2739
2740 long result;
2741 try {
2742 result = region.incrementColumnValue(row1, fam1, qual1, 1, true);
2743 fail("Expected to fail here");
2744 } catch (Exception exception) {
2745
2746 }
2747
2748
2749 assertICV(row1, fam1, qual1, row1Field1);
2750 assertICV(row1, fam1, qual2, row1Field2);
2751 } finally {
2752 HRegion.closeHRegion(this.region);
2753 this.region = null;
2754 }
2755 }
2756
2757 public void testIncrement_WrongInitialSize() throws IOException {
2758 this.region = initHRegion(tableName, getName(), conf, fam1);
2759 try {
2760 byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
2761 long row1Field1 = 0;
2762 int row1Field2 = 1;
2763 Put put1 = new Put(row1);
2764 put1.add(fam1, qual1, Bytes.toBytes(row1Field1));
2765 put1.add(fam1, qual2, Bytes.toBytes(row1Field2));
2766 region.put(put1);
2767 Increment increment = new Increment(row1);
2768 increment.addColumn(fam1, qual1, 1);
2769
2770
2771 region.increment(increment, null, true);
2772 assertICV(row1, fam1, qual1, row1Field1 + 1);
2773
2774
2775 increment = new Increment(row1);
2776 increment.addColumn(fam1, qual2, 1);
2777 try {
2778 region.increment(increment, null, true);
2779 fail("Expected to fail here");
2780 } catch (Exception exception) {
2781
2782 }
2783 assertICV(row1, fam1, qual2, row1Field2);
2784 } finally {
2785 HRegion.closeHRegion(this.region);
2786 this.region = null;
2787 }
2788 }
2789 private void assertICV(byte [] row,
2790 byte [] familiy,
2791 byte[] qualifier,
2792 long amount) throws IOException {
2793
2794 Get get = new Get(row);
2795 get.addColumn(familiy, qualifier);
2796 Result result = region.get(get, null);
2797 assertEquals(1, result.size());
2798
2799 KeyValue kv = result.raw()[0];
2800 long r = Bytes.toLong(kv.getValue());
2801 assertEquals(amount, r);
2802 }
2803
2804 private void assertICV(byte [] row,
2805 byte [] familiy,
2806 byte[] qualifier,
2807 int amount) throws IOException {
2808
2809 Get get = new Get(row);
2810 get.addColumn(familiy, qualifier);
2811 Result result = region.get(get, null);
2812 assertEquals(1, result.size());
2813
2814 KeyValue kv = result.raw()[0];
2815 int r = Bytes.toInt(kv.getValue());
2816 assertEquals(amount, r);
2817 }
2818
2819 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions()
2820 throws IOException {
2821 byte [] tableName = Bytes.toBytes("testtable");
2822 byte [] row1 = Bytes.toBytes("row1");
2823 byte [] fam1 = Bytes.toBytes("fam1");
2824 byte [] qf1 = Bytes.toBytes("qualifier1");
2825 byte [] qf2 = Bytes.toBytes("quateslifier2");
2826
2827 long ts1 = 1;
2828 long ts2 = ts1 + 1;
2829 long ts3 = ts1 + 2;
2830 long ts4 = ts1 + 3;
2831
2832
2833 String method = this.getName();
2834 this.region = initHRegion(tableName, method, conf, fam1);
2835 try {
2836
2837 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
2838 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
2839 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
2840 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
2841
2842 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
2843 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
2844 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
2845 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
2846
2847 Put put = null;
2848 put = new Put(row1);
2849 put.add(kv14);
2850 put.add(kv24);
2851 region.put(put);
2852 region.flushcache();
2853
2854 put = new Put(row1);
2855 put.add(kv23);
2856 put.add(kv13);
2857 region.put(put);
2858 region.flushcache();
2859
2860 put = new Put(row1);
2861 put.add(kv22);
2862 put.add(kv12);
2863 region.put(put);
2864 region.flushcache();
2865
2866 put = new Put(row1);
2867 put.add(kv21);
2868 put.add(kv11);
2869 region.put(put);
2870
2871
2872 List<KeyValue> expected = new ArrayList<KeyValue>();
2873 expected.add(kv14);
2874 expected.add(kv13);
2875 expected.add(kv12);
2876 expected.add(kv24);
2877 expected.add(kv23);
2878 expected.add(kv22);
2879
2880 Scan scan = new Scan(row1);
2881 int versions = 3;
2882 scan.setMaxVersions(versions);
2883 List<KeyValue> actual = new ArrayList<KeyValue>();
2884 InternalScanner scanner = region.getScanner(scan);
2885
2886 boolean hasNext = scanner.next(actual);
2887 assertEquals(false, hasNext);
2888
2889
2890 for(int i=0; i<expected.size(); i++) {
2891 assertEquals(expected.get(i), actual.get(i));
2892 }
2893 } finally {
2894 HRegion.closeHRegion(this.region);
2895 this.region = null;
2896 }
2897 }
2898
2899
2900
2901
2902
2903
2904
2905 public void testScanner_JoinedScanners() throws IOException {
2906 byte [] tableName = Bytes.toBytes("testTable");
2907 byte [] cf_essential = Bytes.toBytes("essential");
2908 byte [] cf_joined = Bytes.toBytes("joined");
2909 byte [] cf_alpha = Bytes.toBytes("alpha");
2910 this.region = initHRegion(tableName, getName(), conf, cf_essential, cf_joined, cf_alpha);
2911 try {
2912 byte [] row1 = Bytes.toBytes("row1");
2913 byte [] row2 = Bytes.toBytes("row2");
2914 byte [] row3 = Bytes.toBytes("row3");
2915
2916 byte [] col_normal = Bytes.toBytes("d");
2917 byte [] col_alpha = Bytes.toBytes("a");
2918
2919 byte [] filtered_val = Bytes.toBytes(3);
2920
2921 Put put = new Put(row1);
2922 put.add(cf_essential, col_normal, Bytes.toBytes(1));
2923 put.add(cf_joined, col_alpha, Bytes.toBytes(1));
2924 region.put(put);
2925
2926 put = new Put(row2);
2927 put.add(cf_essential, col_alpha, Bytes.toBytes(2));
2928 put.add(cf_joined, col_normal, Bytes.toBytes(2));
2929 put.add(cf_alpha, col_alpha, Bytes.toBytes(2));
2930 region.put(put);
2931
2932 put = new Put(row3);
2933 put.add(cf_essential, col_normal, filtered_val);
2934 put.add(cf_joined, col_normal, filtered_val);
2935 region.put(put);
2936
2937
2938
2939
2940
2941 Scan scan = new Scan();
2942 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
2943 CompareOp.NOT_EQUAL, filtered_val);
2944 scan.setFilter(filter);
2945 scan.setLoadColumnFamiliesOnDemand(true);
2946 InternalScanner s = region.getScanner(scan);
2947
2948 List<KeyValue> results = new ArrayList<KeyValue>();
2949 assertTrue(s.next(results));
2950 assertEquals(results.size(), 1);
2951 results.clear();
2952
2953 assertTrue(s.next(results));
2954 assertEquals(results.size(), 3);
2955 assertTrue("orderCheck", results.get(0).matchingFamily(cf_alpha));
2956 assertTrue("orderCheck", results.get(1).matchingFamily(cf_essential));
2957 assertTrue("orderCheck", results.get(2).matchingFamily(cf_joined));
2958 results.clear();
2959
2960 assertFalse(s.next(results));
2961 assertEquals(results.size(), 0);
2962 } finally {
2963 HRegion.closeHRegion(this.region);
2964 this.region = null;
2965 }
2966 }
2967
2968
2969
2970
2971
2972
2973 public void testScanner_JoinedScannersWithLimits() throws IOException {
2974 final byte [] tableName = Bytes.toBytes("testTable");
2975 final byte [] cf_first = Bytes.toBytes("first");
2976 final byte [] cf_second = Bytes.toBytes("second");
2977
2978 this.region = initHRegion(tableName, getName(), conf, cf_first, cf_second);
2979 try {
2980 final byte [] col_a = Bytes.toBytes("a");
2981 final byte [] col_b = Bytes.toBytes("b");
2982
2983 Put put;
2984
2985 for (int i = 0; i < 10; i++) {
2986 put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
2987 put.add(cf_first, col_a, Bytes.toBytes(i));
2988 if (i < 5) {
2989 put.add(cf_first, col_b, Bytes.toBytes(i));
2990 put.add(cf_second, col_a, Bytes.toBytes(i));
2991 put.add(cf_second, col_b, Bytes.toBytes(i));
2992 }
2993 region.put(put);
2994 }
2995
2996 Scan scan = new Scan();
2997 scan.setLoadColumnFamiliesOnDemand(true);
2998 Filter bogusFilter = new FilterBase() {
2999 @Override
3000 public boolean isFamilyEssential(byte[] name) {
3001 return Bytes.equals(name, cf_first);
3002 }
3003 @Override
3004 public void readFields(DataInput arg0) throws IOException {
3005 }
3006
3007 @Override
3008 public void write(DataOutput arg0) throws IOException {
3009 }
3010 };
3011
3012 scan.setFilter(bogusFilter);
3013 InternalScanner s = region.getScanner(scan);
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044 List<KeyValue> results = new ArrayList<KeyValue>();
3045 int index = 0;
3046 while (true) {
3047 boolean more = s.next(results, 3);
3048 if ((index >> 1) < 5) {
3049 if (index % 2 == 0)
3050 assertEquals(results.size(), 3);
3051 else
3052 assertEquals(results.size(), 1);
3053 }
3054 else
3055 assertEquals(results.size(), 1);
3056 results.clear();
3057 index++;
3058 if (!more) break;
3059 }
3060 } finally {
3061 HRegion.closeHRegion(this.region);
3062 this.region = null;
3063 }
3064 }
3065
3066
3067
3068
3069
3070
3071
3072
3073 public void testBasicSplit() throws Exception {
3074 byte [] tableName = Bytes.toBytes("testtable");
3075 byte [][] families = {fam1, fam2, fam3};
3076
3077 Configuration hc = initSplit();
3078
3079 String method = this.getName();
3080 this.region = initHRegion(tableName, method, hc, families);
3081
3082 try {
3083 LOG.info("" + addContent(region, fam3));
3084 region.flushcache();
3085 region.compactStores();
3086 byte [] splitRow = region.checkSplit();
3087 assertNotNull(splitRow);
3088 LOG.info("SplitRow: " + Bytes.toString(splitRow));
3089 HRegion [] regions = splitRegion(region, splitRow);
3090 try {
3091
3092
3093
3094 for (int i = 0; i < regions.length; i++) {
3095 regions[i] = openClosedRegion(regions[i]);
3096 }
3097
3098
3099 assertGet(regions[0], fam3, Bytes.toBytes(START_KEY));
3100 assertGet(regions[1], fam3, splitRow);
3101
3102 assertScan(regions[0], fam3,
3103 Bytes.toBytes(START_KEY));
3104 assertScan(regions[1], fam3, splitRow);
3105
3106 for (int i = 0; i < regions.length; i++) {
3107
3108
3109 for (int j = 0; j < 2; j++) {
3110 addContent(regions[i], fam3);
3111 }
3112 addContent(regions[i], fam2);
3113 addContent(regions[i], fam1);
3114 regions[i].flushcache();
3115 }
3116
3117 byte [][] midkeys = new byte [regions.length][];
3118
3119 for (int i = 0; i < regions.length; i++) {
3120 regions[i].compactStores();
3121 midkeys[i] = regions[i].checkSplit();
3122 }
3123
3124 TreeMap<String, HRegion> sortedMap = new TreeMap<String, HRegion>();
3125
3126
3127 for (int i = 0; i < regions.length; i++) {
3128 HRegion[] rs = null;
3129 if (midkeys[i] != null) {
3130 rs = splitRegion(regions[i], midkeys[i]);
3131 for (int j = 0; j < rs.length; j++) {
3132 sortedMap.put(Bytes.toString(rs[j].getRegionName()),
3133 openClosedRegion(rs[j]));
3134 }
3135 }
3136 }
3137 LOG.info("Made 4 regions");
3138
3139
3140 int interval = (LAST_CHAR - FIRST_CHAR) / 3;
3141 byte[] b = Bytes.toBytes(START_KEY);
3142 for (HRegion r : sortedMap.values()) {
3143 assertGet(r, fam3, b);
3144 b[0] += interval;
3145 }
3146 } finally {
3147 for (int i = 0; i < regions.length; i++) {
3148 try {
3149 regions[i].close();
3150 } catch (IOException e) {
3151
3152 }
3153 }
3154 }
3155 } finally {
3156 HRegion.closeHRegion(this.region);
3157 this.region = null;
3158 }
3159 }
3160
3161 public void testSplitRegion() throws IOException {
3162 byte [] tableName = Bytes.toBytes("testtable");
3163 byte [] qualifier = Bytes.toBytes("qualifier");
3164 Configuration hc = initSplit();
3165 int numRows = 10;
3166 byte [][] families = {fam1, fam3};
3167
3168
3169 String method = this.getName();
3170 this.region = initHRegion(tableName, method, hc, families);
3171
3172
3173 int startRow = 100;
3174 putData(startRow, numRows, qualifier, families);
3175 int splitRow = startRow + numRows;
3176 putData(splitRow, numRows, qualifier, families);
3177 region.flushcache();
3178
3179 HRegion [] regions = null;
3180 try {
3181 regions = splitRegion(region, Bytes.toBytes("" + splitRow));
3182
3183 for (int i = 0; i < regions.length; i++) {
3184 regions[i] = openClosedRegion(regions[i]);
3185 }
3186
3187 assertEquals(2, regions.length);
3188
3189
3190
3191 verifyData(regions[0], startRow, numRows, qualifier, families);
3192 verifyData(regions[1], splitRow, numRows, qualifier, families);
3193
3194 } finally {
3195 HRegion.closeHRegion(this.region);
3196 this.region = null;
3197 }
3198 }
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208 public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
3209 byte[] tableName = Bytes.toBytes("testFlushCacheWhileScanning");
3210 byte[] family = Bytes.toBytes("family");
3211 int numRows = 1000;
3212 int flushAndScanInterval = 10;
3213 int compactInterval = 10 * flushAndScanInterval;
3214
3215 String method = "testFlushCacheWhileScanning";
3216 this.region = initHRegion(tableName,method, conf, family);
3217 try {
3218 FlushThread flushThread = new FlushThread();
3219 flushThread.start();
3220
3221 Scan scan = new Scan();
3222 scan.addFamily(family);
3223 scan.setFilter(new SingleColumnValueFilter(family, qual1,
3224 CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(5L))));
3225
3226 int expectedCount = 0;
3227 List<KeyValue> res = new ArrayList<KeyValue>();
3228
3229 boolean toggle=true;
3230 for (long i = 0; i < numRows; i++) {
3231 Put put = new Put(Bytes.toBytes(i));
3232 put.setWriteToWAL(false);
3233 put.add(family, qual1, Bytes.toBytes(i % 10));
3234 region.put(put);
3235
3236 if (i != 0 && i % compactInterval == 0) {
3237
3238 region.compactStores(true);
3239 }
3240
3241 if (i % 10 == 5L) {
3242 expectedCount++;
3243 }
3244
3245 if (i != 0 && i % flushAndScanInterval == 0) {
3246 res.clear();
3247 InternalScanner scanner = region.getScanner(scan);
3248 if (toggle) {
3249 flushThread.flush();
3250 }
3251 while (scanner.next(res)) ;
3252 if (!toggle) {
3253 flushThread.flush();
3254 }
3255 assertEquals("i=" + i, expectedCount, res.size());
3256 toggle = !toggle;
3257 }
3258 }
3259
3260 flushThread.done();
3261 flushThread.join();
3262 flushThread.checkNoError();
3263 } finally {
3264 try {
3265 HRegion.closeHRegion(this.region);
3266 } catch (DroppedSnapshotException dse) {
3267
3268
3269
3270
3271 }
3272 this.region = null;
3273 }
3274 }
3275
3276 protected class FlushThread extends Thread {
3277 private volatile boolean done;
3278 private Throwable error = null;
3279
3280 public void done() {
3281 done = true;
3282 synchronized (this) {
3283 interrupt();
3284 }
3285 }
3286
3287 public void checkNoError() {
3288 if (error != null) {
3289 assertNull(error);
3290 }
3291 }
3292
3293 @Override
3294 public void run() {
3295 done = false;
3296 while (!done) {
3297 synchronized (this) {
3298 try {
3299 wait();
3300 } catch (InterruptedException ignored) {
3301 if (done) {
3302 break;
3303 }
3304 }
3305 }
3306 try {
3307 region.flushcache();
3308 } catch (IOException e) {
3309 if (!done) {
3310 LOG.error("Error while flusing cache", e);
3311 error = e;
3312 }
3313 break;
3314 }
3315 }
3316
3317 }
3318
3319 public void flush() {
3320 synchronized (this) {
3321 notify();
3322 }
3323
3324 }
3325 }
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335 public void testWritesWhileScanning()
3336 throws IOException, InterruptedException {
3337 byte[] tableName = Bytes.toBytes("testWritesWhileScanning");
3338 int testCount = 100;
3339 int numRows = 1;
3340 int numFamilies = 10;
3341 int numQualifiers = 100;
3342 int flushInterval = 7;
3343 int compactInterval = 5 * flushInterval;
3344 byte[][] families = new byte[numFamilies][];
3345 for (int i = 0; i < numFamilies; i++) {
3346 families[i] = Bytes.toBytes("family" + i);
3347 }
3348 byte[][] qualifiers = new byte[numQualifiers][];
3349 for (int i = 0; i < numQualifiers; i++) {
3350 qualifiers[i] = Bytes.toBytes("qual" + i);
3351 }
3352
3353 String method = "testWritesWhileScanning";
3354 this.region = initHRegion(tableName, method, conf, families);
3355 try {
3356 PutThread putThread = new PutThread(numRows, families, qualifiers);
3357 putThread.start();
3358 putThread.waitForFirstPut();
3359
3360 FlushThread flushThread = new FlushThread();
3361 flushThread.start();
3362
3363 Scan scan = new Scan(Bytes.toBytes("row0"), Bytes.toBytes("row1"));
3364
3365
3366
3367 int expectedCount = numFamilies * numQualifiers;
3368 List<KeyValue> res = new ArrayList<KeyValue>();
3369
3370 long prevTimestamp = 0L;
3371 for (int i = 0; i < testCount; i++) {
3372
3373 if (i != 0 && i % compactInterval == 0) {
3374 region.compactStores(true);
3375 }
3376
3377 if (i != 0 && i % flushInterval == 0) {
3378
3379 flushThread.flush();
3380 }
3381
3382 boolean previousEmpty = res.isEmpty();
3383 res.clear();
3384 InternalScanner scanner = region.getScanner(scan);
3385 while (scanner.next(res)) ;
3386 if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
3387 assertEquals("i=" + i, expectedCount, res.size());
3388 long timestamp = res.get(0).getTimestamp();
3389 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
3390 timestamp >= prevTimestamp);
3391 prevTimestamp = timestamp;
3392 }
3393 }
3394
3395 putThread.done();
3396
3397 region.flushcache();
3398
3399 putThread.join();
3400 putThread.checkNoError();
3401
3402 flushThread.done();
3403 flushThread.join();
3404 flushThread.checkNoError();
3405 } finally {
3406 try {
3407 HRegion.closeHRegion(this.region);
3408 } catch (DroppedSnapshotException dse) {
3409
3410
3411
3412
3413 }
3414 this.region = null;
3415 }
3416 }
3417
3418 protected class PutThread extends Thread {
3419 private volatile boolean done;
3420 private volatile int numPutsFinished = 0;
3421
3422 private Throwable error = null;
3423 private int numRows;
3424 private byte[][] families;
3425 private byte[][] qualifiers;
3426
3427 private PutThread(int numRows, byte[][] families,
3428 byte[][] qualifiers) {
3429 this.numRows = numRows;
3430 this.families = families;
3431 this.qualifiers = qualifiers;
3432 }
3433
3434
3435
3436
3437 public void waitForFirstPut() throws InterruptedException {
3438
3439 while (numPutsFinished == 0) {
3440 checkNoError();
3441 Thread.sleep(50);
3442 }
3443 }
3444
3445 public void done() {
3446 done = true;
3447 synchronized (this) {
3448 interrupt();
3449 }
3450 }
3451
3452 public void checkNoError() {
3453 if (error != null) {
3454 assertNull(error);
3455 }
3456 }
3457
3458 @Override
3459 public void run() {
3460 done = false;
3461 while (!done) {
3462 try {
3463 for (int r = 0; r < numRows; r++) {
3464 byte[] row = Bytes.toBytes("row" + r);
3465 Put put = new Put(row);
3466 put.setWriteToWAL(false);
3467 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
3468 for (byte[] family : families) {
3469 for (byte[] qualifier : qualifiers) {
3470 put.add(family, qualifier, (long) numPutsFinished, value);
3471 }
3472 }
3473
3474 region.put(put);
3475 numPutsFinished++;
3476 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
3477 System.out.println("put iteration = " + numPutsFinished);
3478 Delete delete = new Delete(row, (long)numPutsFinished-30, null);
3479 region.delete(delete, null, true);
3480 }
3481 numPutsFinished++;
3482 }
3483 } catch (InterruptedIOException e) {
3484
3485 } catch (IOException e) {
3486 LOG.error("error while putting records", e);
3487 error = e;
3488 break;
3489 }
3490 }
3491
3492 }
3493
3494 }
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504 public void testWritesWhileGetting()
3505 throws Exception {
3506 byte[] tableName = Bytes.toBytes("testWritesWhileGetting");
3507 int testCount = 100;
3508 int numRows = 1;
3509 int numFamilies = 10;
3510 int numQualifiers = 100;
3511 int compactInterval = 100;
3512 byte[][] families = new byte[numFamilies][];
3513 for (int i = 0; i < numFamilies; i++) {
3514 families[i] = Bytes.toBytes("family" + i);
3515 }
3516 byte[][] qualifiers = new byte[numQualifiers][];
3517 for (int i = 0; i < numQualifiers; i++) {
3518 qualifiers[i] = Bytes.toBytes("qual" + i);
3519 }
3520
3521 Configuration conf = HBaseConfiguration.create(this.conf);
3522
3523 String method = "testWritesWhileGetting";
3524
3525
3526
3527 conf.setInt("hbase.hstore.compaction.min", 1);
3528 conf.setInt("hbase.hstore.compaction.max", 1000);
3529 this.region = initHRegion(tableName, method, conf, families);
3530 PutThread putThread = null;
3531 MultithreadedTestUtil.TestContext ctx =
3532 new MultithreadedTestUtil.TestContext(conf);
3533 try {
3534 putThread = new PutThread(numRows, families, qualifiers);
3535 putThread.start();
3536 putThread.waitForFirstPut();
3537
3538
3539 ctx.addThread(new RepeatingTestThread(ctx) {
3540 private int flushesSinceCompact = 0;
3541 private final int maxFlushesSinceCompact = 20;
3542 public void doAnAction() throws Exception {
3543 if (region.flushcache().isCompactionNeeded()) {
3544 ++flushesSinceCompact;
3545 }
3546
3547 if (flushesSinceCompact == maxFlushesSinceCompact) {
3548 region.compactStores(false);
3549 flushesSinceCompact = 0;
3550 }
3551 }
3552 });
3553 ctx.startThreads();
3554
3555 Get get = new Get(Bytes.toBytes("row0"));
3556 Result result = null;
3557
3558 int expectedCount = numFamilies * numQualifiers;
3559
3560 long prevTimestamp = 0L;
3561 for (int i = 0; i < testCount; i++) {
3562
3563 boolean previousEmpty = result == null || result.isEmpty();
3564 result = region.get(get, null);
3565 if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
3566 assertEquals("i=" + i, expectedCount, result.size());
3567
3568
3569 long timestamp = 0;
3570 for (KeyValue kv : result.raw()) {
3571 if (Bytes.equals(kv.getFamily(), families[0])
3572 && Bytes.equals(kv.getQualifier(), qualifiers[0])) {
3573 timestamp = kv.getTimestamp();
3574 }
3575 }
3576 assertTrue(timestamp >= prevTimestamp);
3577 prevTimestamp = timestamp;
3578 KeyValue previousKV = null;
3579
3580 for (KeyValue kv : result.raw()) {
3581 byte[] thisValue = kv.getValue();
3582 if (previousKV != null) {
3583 if (Bytes.compareTo(previousKV.getValue(), thisValue) != 0) {
3584 LOG.warn("These two KV should have the same value." +
3585 " Previous KV:" +
3586 previousKV + "(memStoreTS:" + previousKV.getMemstoreTS() + ")" +
3587 ", New KV: " +
3588 kv + "(memStoreTS:" + kv.getMemstoreTS() + ")"
3589 );
3590 assertEquals(0, Bytes.compareTo(previousKV.getValue(), thisValue));
3591 }
3592 }
3593 previousKV = kv;
3594 }
3595 }
3596 }
3597 } finally {
3598 if (putThread != null) putThread.done();
3599
3600 region.flushcache();
3601
3602 if (putThread != null) {
3603 putThread.join();
3604 putThread.checkNoError();
3605 }
3606
3607 ctx.stop();
3608 HRegion.closeHRegion(this.region);
3609 this.region = null;
3610 }
3611 }
3612
3613 public void testHolesInMeta() throws Exception {
3614 String method = "testHolesInMeta";
3615 byte[] tableName = Bytes.toBytes(method);
3616 byte[] family = Bytes.toBytes("family");
3617 this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
3618 conf, false, family);
3619 try {
3620 byte[] rowNotServed = Bytes.toBytes("a");
3621 Get g = new Get(rowNotServed);
3622 try {
3623 region.get(g, null);
3624 fail();
3625 } catch (WrongRegionException x) {
3626
3627 }
3628 byte[] row = Bytes.toBytes("y");
3629 g = new Get(row);
3630 region.get(g, null);
3631 } finally {
3632 HRegion.closeHRegion(this.region);
3633 this.region = null;
3634 }
3635 }
3636
3637
3638
3639
3640
3641
3642
3643 @Test
3644 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
3645 HRegionInfo info = null;
3646 try {
3647 FileSystem fs = Mockito.mock(FileSystem.class);
3648 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
3649 HTableDescriptor htd = new HTableDescriptor(tableName);
3650 htd.addFamily(new HColumnDescriptor("cf"));
3651 info = new HRegionInfo(htd.getName(), HConstants.EMPTY_BYTE_ARRAY,
3652 HConstants.EMPTY_BYTE_ARRAY, false);
3653 Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
3654
3655
3656 SchemaMetrics.setUseTableNameInTest(false);
3657 region = HRegion.newHRegion(path, null, fs, conf, info, htd, null);
3658
3659 region.initialize();
3660 fail("Region initialization should fail due to IOException");
3661 } catch (IOException io) {
3662 List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
3663 for (MonitoredTask monitoredTask : tasks) {
3664 if (!(monitoredTask instanceof MonitoredRPCHandler)
3665 && monitoredTask.getDescription().contains(region.toString())) {
3666 assertTrue("Region state should be ABORTED.",
3667 monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
3668 break;
3669 }
3670 }
3671 } finally {
3672 HRegion.closeHRegion(region);
3673 }
3674 }
3675
3676 public void testIndexesScanWithOneDeletedRow() throws IOException {
3677 byte[] tableName = Bytes.toBytes("testIndexesScanWithOneDeletedRow");
3678 byte[] family = Bytes.toBytes("family");
3679
3680
3681 String method = "testIndexesScanWithOneDeletedRow";
3682 this.region = initHRegion(tableName, method, conf, family);
3683 try {
3684 Put put = new Put(Bytes.toBytes(1L));
3685 put.add(family, qual1, 1L, Bytes.toBytes(1L));
3686 region.put(put);
3687
3688 region.flushcache();
3689
3690 Delete delete = new Delete(Bytes.toBytes(1L), 1L, null);
3691
3692 region.delete(delete, null, true);
3693
3694 put = new Put(Bytes.toBytes(2L));
3695 put.add(family, qual1, 2L, Bytes.toBytes(2L));
3696 region.put(put);
3697
3698 Scan idxScan = new Scan();
3699 idxScan.addFamily(family);
3700 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
3701 Arrays.<Filter>asList(new SingleColumnValueFilter(family, qual1,
3702 CompareOp.GREATER_OR_EQUAL,
3703 new BinaryComparator(Bytes.toBytes(0L))),
3704 new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL,
3705 new BinaryComparator(Bytes.toBytes(3L)))
3706 )));
3707 InternalScanner scanner = region.getScanner(idxScan);
3708 List<KeyValue> res = new ArrayList<KeyValue>();
3709
3710
3711 while (scanner.next(res)) ;
3712
3713
3714 assertEquals(1L, res.size());
3715 } finally {
3716 HRegion.closeHRegion(this.region);
3717 this.region = null;
3718 }
3719 }
3720
3721
3722
3723
3724 public void testBloomFilterSize() throws IOException {
3725 byte [] tableName = Bytes.toBytes("testBloomFilterSize");
3726 byte [] row1 = Bytes.toBytes("row1");
3727 byte [] fam1 = Bytes.toBytes("fam1");
3728 byte [] qf1 = Bytes.toBytes("col");
3729 byte [] val1 = Bytes.toBytes("value1");
3730
3731 HColumnDescriptor hcd = new HColumnDescriptor(fam1)
3732 .setMaxVersions(Integer.MAX_VALUE)
3733 .setBloomFilterType(BloomType.ROWCOL);
3734
3735 HTableDescriptor htd = new HTableDescriptor(tableName);
3736 htd.addFamily(hcd);
3737 HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
3738 Path path = new Path(DIR + "testBloomFilterSize");
3739 this.region = HRegion.createHRegion(info, path, conf, htd);
3740 try {
3741 int num_unique_rows = 10;
3742 int duplicate_multiplier =2;
3743 int num_storefiles = 4;
3744
3745 int version = 0;
3746 for (int f =0 ; f < num_storefiles; f++) {
3747 for (int i = 0; i < duplicate_multiplier; i ++) {
3748 for (int j = 0; j < num_unique_rows; j++) {
3749 Put put = new Put(Bytes.toBytes("row" + j));
3750 put.setWriteToWAL(false);
3751 put.add(fam1, qf1, version++, val1);
3752 region.put(put);
3753 }
3754 }
3755 region.flushcache();
3756 }
3757
3758 Store store = region.getStore(fam1);
3759 List<StoreFile> storeFiles = store.getStorefiles();
3760 for (StoreFile storefile : storeFiles) {
3761 StoreFile.Reader reader = storefile.getReader();
3762 reader.loadFileInfo();
3763 reader.loadBloomfilter();
3764 assertEquals(num_unique_rows*duplicate_multiplier, reader.getEntries());
3765 assertEquals(num_unique_rows, reader.getFilterEntries());
3766 }
3767
3768 region.compactStores(true);
3769
3770
3771 storeFiles = store.getStorefiles();
3772 for (StoreFile storefile : storeFiles) {
3773 StoreFile.Reader reader = storefile.getReader();
3774 reader.loadFileInfo();
3775 reader.loadBloomfilter();
3776 assertEquals(num_unique_rows*duplicate_multiplier*num_storefiles,
3777 reader.getEntries());
3778 assertEquals(num_unique_rows, reader.getFilterEntries());
3779 }
3780 } finally {
3781 HRegion.closeHRegion(this.region);
3782 this.region = null;
3783 }
3784 }
3785
3786 public void testAllColumnsWithBloomFilter() throws IOException {
3787 byte [] TABLE = Bytes.toBytes("testAllColumnsWithBloomFilter");
3788 byte [] FAMILY = Bytes.toBytes("family");
3789
3790
3791 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
3792 .setMaxVersions(Integer.MAX_VALUE)
3793 .setBloomFilterType(BloomType.ROWCOL);
3794 HTableDescriptor htd = new HTableDescriptor(TABLE);
3795 htd.addFamily(hcd);
3796 HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
3797 Path path = new Path(DIR + "testAllColumnsWithBloomFilter");
3798 this.region = HRegion.createHRegion(info, path, conf, htd);
3799 try {
3800
3801 byte row[] = Bytes.toBytes("row:" + 0);
3802 byte column[] = Bytes.toBytes("column:" + 0);
3803 Put put = new Put(row);
3804 put.setWriteToWAL(false);
3805 for (long idx = 1; idx <= 4; idx++) {
3806 put.add(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
3807 }
3808 region.put(put);
3809
3810
3811 region.flushcache();
3812
3813
3814 Get get = new Get(row);
3815 get.setMaxVersions();
3816 KeyValue[] kvs = region.get(get, null).raw();
3817
3818
3819 assertEquals(4, kvs.length);
3820 checkOneCell(kvs[0], FAMILY, 0, 0, 4);
3821 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
3822 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
3823 checkOneCell(kvs[3], FAMILY, 0, 0, 1);
3824 } finally {
3825 HRegion.closeHRegion(this.region);
3826 this.region = null;
3827 }
3828 }
3829
3830
3831
3832
3833
3834
3835 public void testDeleteRowWithBloomFilter() throws IOException {
3836 byte [] tableName = Bytes.toBytes("testDeleteRowWithBloomFilter");
3837 byte [] familyName = Bytes.toBytes("familyName");
3838
3839
3840 HColumnDescriptor hcd = new HColumnDescriptor(familyName)
3841 .setMaxVersions(Integer.MAX_VALUE)
3842 .setBloomFilterType(BloomType.ROWCOL);
3843
3844 HTableDescriptor htd = new HTableDescriptor(tableName);
3845 htd.addFamily(hcd);
3846 HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
3847 Path path = new Path(DIR + "TestDeleteRowWithBloomFilter");
3848 this.region = HRegion.createHRegion(info, path, conf, htd);
3849 try {
3850
3851 byte row[] = Bytes.toBytes("row1");
3852 byte col[] = Bytes.toBytes("col1");
3853
3854 Put put = new Put(row);
3855 put.add(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
3856 region.put(put);
3857 region.flushcache();
3858
3859 Delete del = new Delete(row);
3860 region.delete(del, null, true);
3861 region.flushcache();
3862
3863
3864 Get get = new Get(row);
3865 get.addColumn(familyName, col);
3866
3867 KeyValue[] keyValues = region.get(get, null).raw();
3868 assertTrue(keyValues.length == 0);
3869 } finally {
3870 HRegion.closeHRegion(this.region);
3871 this.region = null;
3872 }
3873 }
3874
3875 @Test public void testgetHDFSBlocksDistribution() throws Exception {
3876 HBaseTestingUtility htu = new HBaseTestingUtility();
3877 final int DEFAULT_BLOCK_SIZE = 1024;
3878 htu.getConfiguration().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
3879 htu.getConfiguration().setInt("dfs.replication", 2);
3880
3881
3882
3883 MiniHBaseCluster cluster = null;
3884 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
3885 int regionServersCount = 3;
3886
3887 try {
3888 cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts);
3889 byte [][] families = {fam1, fam2};
3890 HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families);
3891
3892
3893 byte row[] = Bytes.toBytes("row1");
3894 byte col[] = Bytes.toBytes("col1");
3895
3896 Put put = new Put(row);
3897 put.add(fam1, col, 1, Bytes.toBytes("test1"));
3898 put.add(fam2, col, 1, Bytes.toBytes("test2"));
3899 ht.put(put);
3900
3901 HRegion firstRegion = htu.getHBaseCluster().
3902 getRegions(Bytes.toBytes(this.getName())).get(0);
3903 firstRegion.flushcache();
3904 HDFSBlocksDistribution blocksDistribution1 =
3905 firstRegion.getHDFSBlocksDistribution();
3906
3907
3908
3909
3910
3911 long uniqueBlocksWeight1 =
3912 blocksDistribution1.getUniqueBlocksTotalWeight();
3913
3914 String topHost = blocksDistribution1.getTopHosts().get(0);
3915 long topHostWeight = blocksDistribution1.getWeight(topHost);
3916 assertTrue(uniqueBlocksWeight1 == topHostWeight);
3917
3918
3919
3920 HDFSBlocksDistribution blocksDistribution2 =
3921 HRegion.computeHDFSBlocksDistribution(htu.getConfiguration(),
3922 firstRegion.getTableDesc(),
3923 firstRegion.getRegionInfo().getEncodedName());
3924 long uniqueBlocksWeight2 =
3925 blocksDistribution2.getUniqueBlocksTotalWeight();
3926
3927 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
3928
3929 ht.close();
3930 } finally {
3931 if (cluster != null) {
3932 htu.shutdownMiniCluster();
3933 }
3934 }
3935 }
3936
3937
3938
3939
3940
3941 public void testPutWithMemStoreFlush() throws Exception {
3942 Configuration conf = HBaseConfiguration.create();
3943 String method = "testPutWithMemStoreFlush";
3944 byte[] tableName = Bytes.toBytes(method);
3945 byte[] family = Bytes.toBytes("family");;
3946 byte[] qualifier = Bytes.toBytes("qualifier");
3947 byte[] row = Bytes.toBytes("putRow");
3948 byte[] value = null;
3949 this.region = initHRegion(tableName, method, conf, family);
3950 Put put = null;
3951 Get get = null;
3952 List<KeyValue> kvs = null;
3953 Result res = null;
3954
3955 put = new Put(row);
3956 value = Bytes.toBytes("value0");
3957 put.add(family, qualifier, 1234567l, value);
3958 region.put(put);
3959 get = new Get(row);
3960 get.addColumn(family, qualifier);
3961 get.setMaxVersions();
3962 res = this.region.get(get, null);
3963 kvs = res.getColumn(family, qualifier);
3964 assertEquals(1, kvs.size());
3965 assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue());
3966
3967 region.flushcache();
3968 get = new Get(row);
3969 get.addColumn(family, qualifier);
3970 get.setMaxVersions();
3971 res = this.region.get(get, null);
3972 kvs = res.getColumn(family, qualifier);
3973 assertEquals(1, kvs.size());
3974 assertEquals(Bytes.toBytes("value0"), kvs.get(0).getValue());
3975
3976 put = new Put(row);
3977 value = Bytes.toBytes("value1");
3978 put.add(family, qualifier, 1234567l, value);
3979 region.put(put);
3980 get = new Get(row);
3981 get.addColumn(family, qualifier);
3982 get.setMaxVersions();
3983 res = this.region.get(get, null);
3984 kvs = res.getColumn(family, qualifier);
3985 assertEquals(1, kvs.size());
3986 assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue());
3987
3988 region.flushcache();
3989 get = new Get(row);
3990 get.addColumn(family, qualifier);
3991 get.setMaxVersions();
3992 res = this.region.get(get, null);
3993 kvs = res.getColumn(family, qualifier);
3994 assertEquals(1, kvs.size());
3995 assertEquals(Bytes.toBytes("value1"), kvs.get(0).getValue());
3996 }
3997
3998
3999
4000
4001
4002 private static class Incrementer implements Runnable {
4003 private HRegion region;
4004 private final static byte[] incRow = Bytes.toBytes("incRow");
4005 private final static byte[] family = Bytes.toBytes("family");
4006 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4007 private final static long ONE = 1l;
4008 private int incCounter;
4009
4010 public Incrementer(HRegion region, int incCounter) {
4011 this.region = region;
4012 this.incCounter = incCounter;
4013 }
4014
4015 @Override
4016 public void run() {
4017 int count = 0;
4018 while (count < incCounter) {
4019 Increment inc = new Increment(incRow);
4020 inc.addColumn(family, qualifier, ONE);
4021 count++;
4022 try {
4023 region.increment(inc, null, true);
4024 } catch (IOException e) {
4025 e.printStackTrace();
4026 break;
4027 }
4028 }
4029 }
4030 }
4031
4032
4033
4034
4035
4036 private static class Appender implements Runnable {
4037 private HRegion region;
4038 private final static byte[] appendRow = Bytes.toBytes("appendRow");
4039 private final static byte[] family = Bytes.toBytes("family");
4040 private final static byte[] qualifier = Bytes.toBytes("qualifier");
4041 private final static byte[] CHAR = Bytes.toBytes("a");
4042 private int appendCounter;
4043
4044 public Appender(HRegion region, int appendCounter) {
4045 this.region = region;
4046 this.appendCounter = appendCounter;
4047 }
4048
4049 @Override
4050 public void run() {
4051 int count = 0;
4052 while (count < appendCounter) {
4053 Append app = new Append(appendRow);
4054 app.add(family, qualifier, CHAR);
4055 count++;
4056 try {
4057 region.append(app, null, true);
4058 } catch (IOException e) {
4059 e.printStackTrace();
4060 break;
4061 }
4062 }
4063 }
4064 }
4065
4066
4067
4068
4069
4070
4071 @Test
4072 public void testParallelAppendWithMemStoreFlush() throws Exception {
4073 Configuration conf = HBaseConfiguration.create();
4074 String method = "testParallelAppendWithMemStoreFlush";
4075 byte[] tableName = Bytes.toBytes(method);
4076 byte[] family = Appender.family;
4077 this.region = initHRegion(tableName, method, conf, family);
4078 final HRegion region = this.region;
4079 final AtomicBoolean appendDone = new AtomicBoolean(false);
4080 Runnable flusher = new Runnable() {
4081 @Override
4082 public void run() {
4083 while (!appendDone.get()) {
4084 try {
4085 region.flushcache();
4086 } catch (Exception e) {
4087 e.printStackTrace();
4088 }
4089 }
4090 }
4091 };
4092
4093
4094 int threadNum = 20;
4095 int appendCounter = 100;
4096 byte[] expected = new byte[threadNum * appendCounter];
4097 for (int i = 0; i < threadNum * appendCounter; i++) {
4098 System.arraycopy(Appender.CHAR, 0, expected, i, 1);
4099 }
4100 Thread[] appenders = new Thread[threadNum];
4101 Thread flushThread = new Thread(flusher);
4102 for (int i = 0; i < threadNum; i++) {
4103 appenders[i] = new Thread(new Appender(this.region, appendCounter));
4104 appenders[i].start();
4105 }
4106 flushThread.start();
4107 for (int i = 0; i < threadNum; i++) {
4108 appenders[i].join();
4109 }
4110
4111 appendDone.set(true);
4112 flushThread.join();
4113
4114 Get get = new Get(Appender.appendRow);
4115 get.addColumn(Appender.family, Appender.qualifier);
4116 get.setMaxVersions(1);
4117 Result res = this.region.get(get, null);
4118 List<KeyValue> kvs = res.getColumn(Appender.family, Appender.qualifier);
4119
4120
4121 assertEquals(kvs.size(), 1);
4122 KeyValue kv = kvs.get(0);
4123 byte[] appendResult = new byte[kv.getValueLength()];
4124 System.arraycopy(kv.getBuffer(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
4125 assertEquals(expected, appendResult);
4126 this.region = null;
4127 }
4128
4129
4130
4131
4132
4133 @Test
4134 public void testParallelIncrementWithMemStoreFlush() throws Exception {
4135 String method = "testParallelIncrementWithMemStoreFlush";
4136 byte[] tableName = Bytes.toBytes(method);
4137 byte[] family = Incrementer.family;
4138 this.region = initHRegion(tableName, method, conf, family);
4139 final HRegion region = this.region;
4140 final AtomicBoolean incrementDone = new AtomicBoolean(false);
4141 Runnable reader = new Runnable() {
4142 @Override
4143 public void run() {
4144 while (!incrementDone.get()) {
4145 try {
4146 region.flushcache();
4147 } catch (Exception e) {
4148 e.printStackTrace();
4149 }
4150 }
4151 }
4152 };
4153
4154
4155 int threadNum = 20;
4156 int incCounter = 100;
4157 long expected = threadNum * incCounter;
4158 Thread[] incrementers = new Thread[threadNum];
4159 Thread flushThread = new Thread(reader);
4160 for (int i = 0; i < threadNum; i++) {
4161 incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
4162 incrementers[i].start();
4163 }
4164 flushThread.start();
4165 for (int i = 0; i < threadNum; i++) {
4166 incrementers[i].join();
4167 }
4168
4169 incrementDone.set(true);
4170 flushThread.join();
4171
4172 Get get = new Get(Incrementer.incRow);
4173 get.addColumn(Incrementer.family, Incrementer.qualifier);
4174 get.setMaxVersions(1);
4175 Result res = this.region.get(get, null);
4176 List<KeyValue> kvs = res.getColumn(Incrementer.family,
4177 Incrementer.qualifier);
4178
4179
4180 assertEquals(kvs.size(), 1);
4181 KeyValue kv = kvs.get(0);
4182 assertEquals(expected, Bytes.toLong(kv.getBuffer(), kv.getValueOffset()));
4183 this.region = null;
4184 }
4185
4186 private void putData(int startRow, int numRows, byte [] qf,
4187 byte [] ...families)
4188 throws IOException {
4189 for(int i=startRow; i<startRow+numRows; i++) {
4190 Put put = new Put(Bytes.toBytes("" + i));
4191 put.setWriteToWAL(false);
4192 for(byte [] family : families) {
4193 put.add(family, qf, null);
4194 }
4195 region.put(put);
4196 }
4197 }
4198
4199 private void verifyData(HRegion newReg, int startRow, int numRows, byte [] qf,
4200 byte [] ... families)
4201 throws IOException {
4202 for(int i=startRow; i<startRow + numRows; i++) {
4203 byte [] row = Bytes.toBytes("" + i);
4204 Get get = new Get(row);
4205 for(byte [] family : families) {
4206 get.addColumn(family, qf);
4207 }
4208 Result result = newReg.get(get, null);
4209 KeyValue [] raw = result.raw();
4210 assertEquals(families.length, result.size());
4211 for(int j=0; j<families.length; j++) {
4212 assertEquals(0, Bytes.compareTo(row, raw[j].getRow()));
4213 assertEquals(0, Bytes.compareTo(families[j], raw[j].getFamily()));
4214 assertEquals(0, Bytes.compareTo(qf, raw[j].getQualifier()));
4215 }
4216 }
4217 }
4218
4219 private void assertGet(final HRegion r, final byte [] family, final byte [] k)
4220 throws IOException {
4221
4222 Get get = new Get(k).addFamily(family).setMaxVersions();
4223 KeyValue [] results = r.get(get, null).raw();
4224 for (int j = 0; j < results.length; j++) {
4225 byte [] tmp = results[j].getValue();
4226
4227 assertTrue(Bytes.equals(k, tmp));
4228 }
4229 }
4230
4231
4232
4233
4234
4235
4236
4237
4238 private void assertScan(final HRegion r, final byte [] fs,
4239 final byte [] firstValue)
4240 throws IOException {
4241 byte [][] families = {fs};
4242 Scan scan = new Scan();
4243 for (int i = 0; i < families.length; i++) scan.addFamily(families[i]);
4244 InternalScanner s = r.getScanner(scan);
4245 try {
4246 List<KeyValue> curVals = new ArrayList<KeyValue>();
4247 boolean first = true;
4248 OUTER_LOOP: while(s.next(curVals)) {
4249 for (KeyValue kv: curVals) {
4250 byte [] val = kv.getValue();
4251 byte [] curval = val;
4252 if (first) {
4253 first = false;
4254 assertTrue(Bytes.compareTo(curval, firstValue) == 0);
4255 } else {
4256
4257 break OUTER_LOOP;
4258 }
4259 }
4260 }
4261 } finally {
4262 s.close();
4263 }
4264 }
4265
4266
4267
4268
4269
4270 @Test
4271 public void testFlushResult() throws IOException {
4272 String method = "testFlushResult";
4273 byte[] tableName = Bytes.toBytes(method);
4274 byte[] family = Bytes.toBytes("family");
4275
4276 this.region = initHRegion(tableName, method, conf, family);
4277
4278
4279 HRegion.FlushResult fr = region.flushcache();
4280 assertFalse(fr.isFlushSucceeded());
4281 assertFalse(fr.isCompactionNeeded());
4282
4283
4284 for (int i = 0; i < 3; i++) {
4285 Put put = new Put(tableName).add(family, family, tableName);
4286 region.put(put);
4287 fr = region.flushcache();
4288 assertTrue(fr.isFlushSucceeded());
4289 assertFalse(fr.isCompactionNeeded());
4290 }
4291
4292
4293 for (int i = 0; i < 2; i++) {
4294 Put put = new Put(tableName).add(family, family, tableName);
4295 region.put(put);
4296 fr = region.flushcache();
4297 assertTrue(fr.isFlushSucceeded());
4298 assertTrue(fr.isCompactionNeeded());
4299 }
4300 }
4301
4302 private Configuration initSplit() {
4303 Configuration conf = HBaseConfiguration.create(this.conf);
4304
4305
4306 conf.setInt("hbase.hstore.compactionThreshold", 2);
4307
4308
4309 conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
4310
4311 conf.setInt(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, 10 * 1000);
4312
4313
4314 conf.setLong("hbase.client.pause", 15 * 1000);
4315
4316
4317
4318 conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
4319 return conf;
4320 }
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330 public static HRegion initHRegion (byte [] tableName, String callingMethod,
4331 Configuration conf, byte [] ... families)
4332 throws IOException{
4333 return initHRegion(tableName, null, null, callingMethod, conf, false, families);
4334 }
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345 public static HRegion initHRegion (byte [] tableName, String callingMethod,
4346 Configuration conf, boolean isReadOnly, byte [] ... families)
4347 throws IOException{
4348 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
4349 }
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
4363 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
4364 throws IOException {
4365 HTableDescriptor htd = new HTableDescriptor(tableName);
4366 htd.setReadOnly(isReadOnly);
4367 for(byte [] family : families) {
4368 htd.addFamily(new HColumnDescriptor(family));
4369 }
4370 HRegionInfo info = new HRegionInfo(htd.getName(), startKey, stopKey, false);
4371 Path path = new Path(DIR + callingMethod);
4372 FileSystem fs = FileSystem.get(conf);
4373 if (fs.exists(path)) {
4374 if (!fs.delete(path, true)) {
4375 throw new IOException("Failed delete of " + path);
4376 }
4377 }
4378 return HRegion.createHRegion(info, path, conf, htd);
4379 }
4380
4381
4382
4383
4384
4385 private void checkOneCell(KeyValue kv, byte[] cf,
4386 int rowIdx, int colIdx, long ts) {
4387 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
4388 assertEquals("Row mismatch which checking: " + ctx,
4389 "row:"+ rowIdx, Bytes.toString(kv.getRow()));
4390 assertEquals("ColumnFamily mismatch while checking: " + ctx,
4391 Bytes.toString(cf), Bytes.toString(kv.getFamily()));
4392 assertEquals("Column qualifier mismatch while checking: " + ctx,
4393 "column:" + colIdx, Bytes.toString(kv.getQualifier()));
4394 assertEquals("Timestamp mismatch while checking: " + ctx,
4395 ts, kv.getTimestamp());
4396 assertEquals("Value mismatch while checking: " + ctx,
4397 "value-version-" + ts, Bytes.toString(kv.getValue()));
4398 }
4399
4400
4401 @org.junit.Rule
4402 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
4403 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
4404 }
4405