1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.Random;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.HBaseTestCase;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Scan;
38 import org.apache.hadoop.hbase.io.hfile.Compression;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.hdfs.MiniDFSCluster;
41
42 public class TestWideScanner extends HBaseTestCase {
43 private final Log LOG = LogFactory.getLog(this.getClass());
44
45 static final byte[] A = Bytes.toBytes("A");
46 static final byte[] B = Bytes.toBytes("B");
47 static final byte[] C = Bytes.toBytes("C");
48 static byte[][] COLUMNS = { A, B, C };
49 static final Random rng = new Random();
50 static final HTableDescriptor TESTTABLEDESC =
51 new HTableDescriptor("testwidescan");
52 static {
53 TESTTABLEDESC.addFamily(new HColumnDescriptor(A,
54 10,
55 Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
56 HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
57 HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
58 TESTTABLEDESC.addFamily(new HColumnDescriptor(B,
59 10,
60 Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
61 HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
62 HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
63 TESTTABLEDESC.addFamily(new HColumnDescriptor(C,
64 10,
65 Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
66 HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
67 HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
68 }
69
70
71 public static final HRegionInfo REGION_INFO =
72 new HRegionInfo(TESTTABLEDESC, HConstants.EMPTY_BYTE_ARRAY,
73 HConstants.EMPTY_BYTE_ARRAY);
74
75 MiniDFSCluster cluster = null;
76 HRegion r;
77
78 @Override
79 public void setUp() throws Exception {
80 cluster = new MiniDFSCluster(conf, 2, true, (String[])null);
81
82 this.conf.set(HConstants.HBASE_DIR,
83 this.cluster.getFileSystem().getHomeDirectory().toString());
84 super.setUp();
85 }
86
87 private int addWideContent(HRegion region) throws IOException {
88 int count = 0;
89 for (char c = 'a'; c <= 'c'; c++) {
90 byte[] row = Bytes.toBytes("ab" + c);
91 int i;
92 for (i = 0; i < 2500; i++) {
93 byte[] b = Bytes.toBytes(String.format("%10d", i));
94 Put put = new Put(row);
95 put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, b);
96 region.put(put);
97 count++;
98 }
99 }
100 return count;
101 }
102
103 public void testWideScanBatching() throws IOException {
104 try {
105 this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
106 int inserted = addWideContent(this.r);
107 List<KeyValue> results = new ArrayList<KeyValue>();
108 Scan scan = new Scan();
109 scan.addFamily(A);
110 scan.addFamily(B);
111 scan.addFamily(C);
112 scan.setBatch(1000);
113 InternalScanner s = r.getScanner(scan);
114 int total = 0;
115 int i = 0;
116 boolean more;
117 do {
118 more = s.next(results);
119 i++;
120 LOG.info("iteration #" + i + ", results.size=" + results.size());
121
122
123 assertTrue(results.size() <= 1000);
124
125 total += results.size();
126
127 if (results.size() > 0) {
128
129 byte[] row = results.get(0).getRow();
130 for (KeyValue kv: results) {
131 assertTrue(Bytes.equals(row, kv.getRow()));
132 }
133 }
134
135 results.clear();
136 } while (more);
137
138
139 LOG.info("inserted " + inserted + ", scanned " + total);
140 assertTrue(total == inserted);
141
142 s.close();
143 } finally {
144 this.r.close();
145 this.r.getLog().closeAndDelete();
146 shutdownDfs(this.cluster);
147 }
148 }
149 }