1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Iterator;
25 import java.util.List;
26 import java.util.Random;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.*;
31 import org.apache.hadoop.hbase.client.Put;
32 import org.apache.hadoop.hbase.client.Scan;
33 import org.apache.hadoop.hbase.client.Durability;
34 import org.apache.hadoop.hbase.io.compress.Compression;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.apache.hadoop.hdfs.MiniDFSCluster;
37 import org.junit.experimental.categories.Category;
38
39 @Category(SmallTests.class)
40 public class TestWideScanner extends HBaseTestCase {
41 private final Log LOG = LogFactory.getLog(this.getClass());
42
43 static final byte[] A = Bytes.toBytes("A");
44 static final byte[] B = Bytes.toBytes("B");
45 static final byte[] C = Bytes.toBytes("C");
46 static byte[][] COLUMNS = { A, B, C };
47 static final Random rng = new Random();
48 static final HTableDescriptor TESTTABLEDESC =
49 new HTableDescriptor(TableName.valueOf("testwidescan"));
50 static {
51 for (byte[] cfName : new byte[][] { A, B, C }) {
52 TESTTABLEDESC.addFamily(new HColumnDescriptor(cfName)
53
54 .setMaxVersions(100)
55 .setBlocksize(8 * 1024)
56 );
57 }
58 }
59
60
61 HRegion r;
62
63 private int addWideContent(HRegion region) throws IOException {
64 int count = 0;
65 for (char c = 'a'; c <= 'c'; c++) {
66 byte[] row = Bytes.toBytes("ab" + c);
67 int i, j;
68 long ts = System.currentTimeMillis();
69 for (i = 0; i < 100; i++) {
70 byte[] b = Bytes.toBytes(String.format("%10d", i));
71 for (j = 0; j < 100; j++) {
72 Put put = new Put(row);
73 put.setDurability(Durability.SKIP_WAL);
74 put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, ++ts, b);
75 region.put(put);
76 count++;
77 }
78 }
79 }
80 return count;
81 }
82
83 public void testWideScanBatching() throws IOException {
84 final int batch = 256;
85 try {
86 this.r = createNewHRegion(TESTTABLEDESC, null, null);
87 int inserted = addWideContent(this.r);
88 List<KeyValue> results = new ArrayList<KeyValue>();
89 Scan scan = new Scan();
90 scan.addFamily(A);
91 scan.addFamily(B);
92 scan.addFamily(C);
93 scan.setMaxVersions(100);
94 scan.setBatch(batch);
95 InternalScanner s = r.getScanner(scan);
96 int total = 0;
97 int i = 0;
98 boolean more;
99 do {
100 more = s.next(results);
101 i++;
102 LOG.info("iteration #" + i + ", results.size=" + results.size());
103
104
105 assertTrue(results.size() <= batch);
106
107 total += results.size();
108
109 if (results.size() > 0) {
110
111 byte[] row = results.get(0).getRow();
112 for (KeyValue kv: results) {
113 assertTrue(Bytes.equals(row, kv.getRow()));
114 }
115 }
116
117 results.clear();
118
119
120 Iterator<KeyValueScanner> scanners =
121 ((HRegion.RegionScannerImpl)s).storeHeap.getHeap().iterator();
122 while (scanners.hasNext()) {
123 StoreScanner ss = (StoreScanner)scanners.next();
124 ss.updateReaders();
125 }
126 } while (more);
127
128
129 LOG.info("inserted " + inserted + ", scanned " + total);
130 assertEquals(total, inserted);
131
132 s.close();
133 } finally {
134 HRegion.closeHRegion(this.r);
135 }
136 }
137
138 }
139