1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.client;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertTrue;
21
22 import java.util.ArrayList;
23 import java.util.List;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.HBaseTestingUtility;
28 import org.apache.hadoop.hbase.HTestConst;
29 import org.apache.hadoop.hbase.KeyValue;
30 import org.apache.hadoop.hbase.MediumTests;
31 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
32 import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
33 import org.apache.hadoop.hbase.util.Bytes;
34 import org.junit.After;
35 import org.junit.AfterClass;
36 import org.junit.Before;
37 import org.junit.BeforeClass;
38 import org.junit.Test;
39 import org.junit.experimental.categories.Category;
40
41
42
43
44 @Category(MediumTests.class)
45 public class TestScannersFromClientSide {
46 private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
47
48 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49 private static byte [] ROW = Bytes.toBytes("testRow");
50 private static byte [] FAMILY = Bytes.toBytes("testFamily");
51 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
52 private static byte [] VALUE = Bytes.toBytes("testValue");
53
54
55
56
57 @BeforeClass
58 public static void setUpBeforeClass() throws Exception {
59 TEST_UTIL.startMiniCluster(3);
60 }
61
62
63
64
65 @AfterClass
66 public static void tearDownAfterClass() throws Exception {
67 TEST_UTIL.shutdownMiniCluster();
68 }
69
70
71
72
73 @Before
74 public void setUp() throws Exception {
75
76 }
77
78
79
80
81 @After
82 public void tearDown() throws Exception {
83
84 }
85
86
87
88
89
90
91 @Test
92 public void testScanBatch() throws Exception {
93 byte [] TABLE = Bytes.toBytes("testScanBatch");
94 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
95
96 HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
97
98 Put put;
99 Scan scan;
100 Delete delete;
101 Result result;
102 ResultScanner scanner;
103 boolean toLog = true;
104 List<KeyValue> kvListExp;
105
106
107 put = new Put(ROW);
108 for (int i=0; i < QUALIFIERS.length; i++) {
109 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
110 put.add(kv);
111 }
112 ht.put(put);
113
114
115 put = new Put(ROW);
116 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
117 put.add(kv);
118 ht.put(put);
119
120
121 delete = new Delete(ROW);
122 delete.deleteFamily(FAMILY, 3);
123 ht.delete(delete);
124
125
126 scan = new Scan(ROW);
127 scan.setMaxVersions();
128 scanner = ht.getScanner(scan);
129
130
131 kvListExp = new ArrayList<KeyValue>();
132 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
133 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
134 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
135 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
136 result = scanner.next();
137 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
138
139
140 scan = new Scan(ROW);
141 scan.setMaxVersions();
142 scan.setBatch(2);
143 scanner = ht.getScanner(scan);
144
145
146 kvListExp = new ArrayList<KeyValue>();
147 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
148 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
149 result = scanner.next();
150 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
151
152
153 kvListExp = new ArrayList<KeyValue>();
154 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
155 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
156 result = scanner.next();
157 verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
158
159 }
160
161
162
163
164
165
166 @Test
167 public void testGetMaxResults() throws Exception {
168 byte [] TABLE = Bytes.toBytes("testGetMaxResults");
169 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
170 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
171
172 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
173
174 Get get;
175 Put put;
176 Result result;
177 boolean toLog = true;
178 List<KeyValue> kvListExp;
179
180 kvListExp = new ArrayList<KeyValue>();
181
182 put = new Put(ROW);
183 for (int i=0; i < 10; i++) {
184 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
185 put.add(kv);
186 kvListExp.add(kv);
187 }
188 ht.put(put);
189
190 get = new Get(ROW);
191 result = ht.get(get);
192 verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
193
194 get = new Get(ROW);
195 get.setMaxResultsPerColumnFamily(2);
196 result = ht.get(get);
197 kvListExp = new ArrayList<KeyValue>();
198 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
199 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
200 verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
201
202
203 get = new Get(ROW);
204 get.setMaxResultsPerColumnFamily(5);
205 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
206 true));
207 result = ht.get(get);
208 kvListExp = new ArrayList<KeyValue>();
209 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
210 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
211 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
212 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
213 verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
214
215
216
217 put = new Put(ROW);
218 for (int i=0; i < QUALIFIERS.length; i++) {
219 KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
220 put.add(kv);
221 }
222 ht.put(put);
223
224 put = new Put(ROW);
225 for (int i=0; i < 10; i++) {
226 KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
227 put.add(kv);
228 }
229 ht.put(put);
230
231 get = new Get(ROW);
232 get.setMaxResultsPerColumnFamily(12);
233 get.addFamily(FAMILIES[1]);
234 get.addFamily(FAMILIES[2]);
235 result = ht.get(get);
236 kvListExp = new ArrayList<KeyValue>();
237
238 for (int i=0; i < 10; i++) {
239 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
240 }
241 for (int i=0; i < 2; i++) {
242 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
243 }
244 for (int i=10; i < 20; i++) {
245 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
246 }
247 verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
248
249
250 get = new Get(ROW);
251 get.setMaxResultsPerColumnFamily(3);
252 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
253 result = ht.get(get);
254 kvListExp = new ArrayList<KeyValue>();
255 for (int i=2; i < 5; i++) {
256 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
257 }
258 for (int i=2; i < 5; i++) {
259 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
260 }
261 for (int i=2; i < 5; i++) {
262 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
263 }
264 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
265
266 get = new Get(ROW);
267 get.setMaxResultsPerColumnFamily(7);
268 get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
269 result = ht.get(get);
270 kvListExp = new ArrayList<KeyValue>();
271 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
272 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
273 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
274 for (int i=10; i < 16; i++) {
275 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
276 }
277 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
278
279 }
280
281
282
283
284
285
286 @Test
287 public void testScanMaxResults() throws Exception {
288 byte [] TABLE = Bytes.toBytes("testScanLimit");
289 byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
290 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
291 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
292
293 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
294
295 Put put;
296 Scan scan;
297 Result result;
298 boolean toLog = true;
299 List<KeyValue> kvListExp, kvListScan;
300
301 kvListExp = new ArrayList<KeyValue>();
302
303 for (int r=0; r < ROWS.length; r++) {
304 put = new Put(ROWS[r]);
305 for (int c=0; c < FAMILIES.length; c++) {
306 for (int q=0; q < QUALIFIERS.length; q++) {
307 KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
308 put.add(kv);
309 if (q < 4) {
310 kvListExp.add(kv);
311 }
312 }
313 }
314 ht.put(put);
315 }
316
317 scan = new Scan();
318 scan.setMaxResultsPerColumnFamily(4);
319 ResultScanner scanner = ht.getScanner(scan);
320 kvListScan = new ArrayList<KeyValue>();
321 while ((result = scanner.next()) != null) {
322 for (KeyValue kv : result.list()) {
323 kvListScan.add(kv);
324 }
325 }
326 result = new Result(kvListScan);
327 verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
328
329 }
330
331
332
333
334
335
336 @Test
337 public void testGetRowOffset() throws Exception {
338 byte [] TABLE = Bytes.toBytes("testGetRowOffset");
339 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
340 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
341
342 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
343
344 Get get;
345 Put put;
346 Result result;
347 boolean toLog = true;
348 List<KeyValue> kvListExp;
349
350
351 kvListExp = new ArrayList<KeyValue>();
352 put = new Put(ROW);
353 for (int i=0; i < 10; i++) {
354 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
355 put.add(kv);
356
357 if (i < 2) continue;
358 kvListExp.add(kv);
359 }
360 ht.put(put);
361
362
363 get = new Get(ROW);
364 get.setRowOffsetPerColumnFamily(2);
365 result = ht.get(get);
366 verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
367
368
369 get = new Get(ROW);
370 get.setRowOffsetPerColumnFamily(20);
371 result = ht.get(get);
372 kvListExp = new ArrayList<KeyValue>();
373 verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
374
375
376 get = new Get(ROW);
377 get.setRowOffsetPerColumnFamily(4);
378 get.setMaxResultsPerColumnFamily(5);
379 result = ht.get(get);
380 kvListExp = new ArrayList<KeyValue>();
381 for (int i=4; i < 9; i++) {
382 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
383 }
384 verifyResult(result, kvListExp, toLog,
385 "Testing offset + setMaxResultsPerCF");
386
387
388 get = new Get(ROW);
389 get.setRowOffsetPerColumnFamily(1);
390 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
391 true));
392 result = ht.get(get);
393 kvListExp = new ArrayList<KeyValue>();
394 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
395 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
396 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
397 verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
398
399
400
401 for(int j=2; j > 0; j--) {
402 put = new Put(ROW);
403 for (int i=0; i < 10; i++) {
404 KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
405 put.add(kv);
406 }
407 ht.put(put);
408 }
409
410 get = new Get(ROW);
411 get.setRowOffsetPerColumnFamily(4);
412 get.setMaxResultsPerColumnFamily(2);
413 get.addFamily(FAMILIES[1]);
414 get.addFamily(FAMILIES[2]);
415 result = ht.get(get);
416 kvListExp = new ArrayList<KeyValue>();
417
418 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
419 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
420 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
421 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
422 verifyResult(result, kvListExp, toLog,
423 "Testing offset + multiple CFs + maxResults");
424 }
425
426 static void verifyResult(Result result, List<KeyValue> expKvList, boolean toLog,
427 String msg) {
428
429 LOG.info(msg);
430 LOG.info("Expected count: " + expKvList.size());
431 LOG.info("Actual count: " + result.size());
432 if (expKvList.size() == 0)
433 return;
434
435 int i = 0;
436 for (KeyValue kv : result.raw()) {
437 if (i >= expKvList.size()) {
438 break;
439 }
440
441 KeyValue kvExp = expKvList.get(i++);
442 if (toLog) {
443 LOG.info("get kv is: " + kv.toString());
444 LOG.info("exp kv is: " + kvExp.toString());
445 }
446 assertTrue("Not equal", kvExp.equals(kv));
447 }
448
449 assertEquals(expKvList.size(), result.size());
450 }
451
452
453 }