1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.client;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertTrue;
21
22 import java.util.ArrayList;
23 import java.util.List;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.Cell;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HRegionInfo;
30 import org.apache.hadoop.hbase.HRegionLocation;
31 import org.apache.hadoop.hbase.HTestConst;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.MiniHBaseCluster;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
37 import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
38 import org.apache.hadoop.hbase.master.HMaster;
39 import org.apache.hadoop.hbase.master.RegionState.State;
40 import org.apache.hadoop.hbase.master.RegionStates;
41 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
42 import org.apache.hadoop.hbase.regionserver.HRegionServer;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.ConfigUtil;
45 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
46 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
47 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
48 import org.junit.After;
49 import org.junit.AfterClass;
50 import org.junit.Before;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55
56
57
58 @Category(MediumTests.class)
59 public class TestScannersFromClientSide {
60 private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
61
62 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63 private static byte [] ROW = Bytes.toBytes("testRow");
64 private static byte [] FAMILY = Bytes.toBytes("testFamily");
65 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
66 private static byte [] VALUE = Bytes.toBytes("testValue");
67
68
69
70
71 @BeforeClass
72 public static void setUpBeforeClass() throws Exception {
73 TEST_UTIL.startMiniCluster(3);
74 }
75
76
77
78
79 @AfterClass
80 public static void tearDownAfterClass() throws Exception {
81 TEST_UTIL.shutdownMiniCluster();
82 }
83
84
85
86
87 @Before
88 public void setUp() throws Exception {
89
90 }
91
92
93
94
95 @After
96 public void tearDown() throws Exception {
97
98 }
99
100
101
102
103
104
105 @Test
106 public void testScanBatch() throws Exception {
107 TableName TABLE = TableName.valueOf("testScanBatch");
108 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
109
110 Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
111
112 Put put;
113 Scan scan;
114 Delete delete;
115 Result result;
116 ResultScanner scanner;
117 boolean toLog = true;
118 List<Cell> kvListExp;
119
120
121 put = new Put(ROW);
122 for (int i=0; i < QUALIFIERS.length; i++) {
123 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
124 put.add(kv);
125 }
126 ht.put(put);
127
128
129 put = new Put(ROW);
130 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
131 put.add(kv);
132 ht.put(put);
133
134
135 delete = new Delete(ROW);
136 delete.deleteFamily(FAMILY, 3);
137 ht.delete(delete);
138
139
140 scan = new Scan(ROW);
141 scan.setMaxVersions();
142 scanner = ht.getScanner(scan);
143
144
145 kvListExp = new ArrayList<Cell>();
146 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
147 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
148 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
149 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
150 result = scanner.next();
151 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
152
153
154 scan = new Scan(ROW);
155 scan.setMaxVersions();
156 scan.setBatch(2);
157 scanner = ht.getScanner(scan);
158
159
160 kvListExp = new ArrayList<Cell>();
161 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
162 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
163 result = scanner.next();
164 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
165
166
167 kvListExp = new ArrayList<Cell>();
168 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
169 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
170 result = scanner.next();
171 verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
172
173 }
174
175
176
177
178
179
180 @Test
181 public void testGetMaxResults() throws Exception {
182 byte [] TABLE = Bytes.toBytes("testGetMaxResults");
183 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
184 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
185
186 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
187
188 Get get;
189 Put put;
190 Result result;
191 boolean toLog = true;
192 List<Cell> kvListExp;
193
194 kvListExp = new ArrayList<Cell>();
195
196 put = new Put(ROW);
197 for (int i=0; i < 10; i++) {
198 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
199 put.add(kv);
200 kvListExp.add(kv);
201 }
202 ht.put(put);
203
204 get = new Get(ROW);
205 result = ht.get(get);
206 verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
207
208 get = new Get(ROW);
209 get.setMaxResultsPerColumnFamily(2);
210 result = ht.get(get);
211 kvListExp = new ArrayList<Cell>();
212 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
213 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
214 verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
215
216
217 get = new Get(ROW);
218 get.setMaxResultsPerColumnFamily(5);
219 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
220 true));
221 result = ht.get(get);
222 kvListExp = new ArrayList<Cell>();
223 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
224 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
225 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
226 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
227 verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
228
229
230
231 put = new Put(ROW);
232 for (int i=0; i < QUALIFIERS.length; i++) {
233 KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
234 put.add(kv);
235 }
236 ht.put(put);
237
238 put = new Put(ROW);
239 for (int i=0; i < 10; i++) {
240 KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
241 put.add(kv);
242 }
243 ht.put(put);
244
245 get = new Get(ROW);
246 get.setMaxResultsPerColumnFamily(12);
247 get.addFamily(FAMILIES[1]);
248 get.addFamily(FAMILIES[2]);
249 result = ht.get(get);
250 kvListExp = new ArrayList<Cell>();
251
252 for (int i=0; i < 10; i++) {
253 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
254 }
255 for (int i=0; i < 2; i++) {
256 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
257 }
258 for (int i=10; i < 20; i++) {
259 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
260 }
261 verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
262
263
264 get = new Get(ROW);
265 get.setMaxResultsPerColumnFamily(3);
266 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
267 result = ht.get(get);
268 kvListExp = new ArrayList<Cell>();
269 for (int i=2; i < 5; i++) {
270 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
271 }
272 for (int i=2; i < 5; i++) {
273 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
274 }
275 for (int i=2; i < 5; i++) {
276 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
277 }
278 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
279
280 get = new Get(ROW);
281 get.setMaxResultsPerColumnFamily(7);
282 get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
283 result = ht.get(get);
284 kvListExp = new ArrayList<Cell>();
285 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
286 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
287 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
288 for (int i=10; i < 16; i++) {
289 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
290 }
291 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
292
293 }
294
295
296
297
298
299
300 @Test
301 public void testScanMaxResults() throws Exception {
302 byte [] TABLE = Bytes.toBytes("testScanLimit");
303 byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
304 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
305 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
306
307 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
308
309 Put put;
310 Scan scan;
311 Result result;
312 boolean toLog = true;
313 List<Cell> kvListExp, kvListScan;
314
315 kvListExp = new ArrayList<Cell>();
316
317 for (int r=0; r < ROWS.length; r++) {
318 put = new Put(ROWS[r]);
319 for (int c=0; c < FAMILIES.length; c++) {
320 for (int q=0; q < QUALIFIERS.length; q++) {
321 KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
322 put.add(kv);
323 if (q < 4) {
324 kvListExp.add(kv);
325 }
326 }
327 }
328 ht.put(put);
329 }
330
331 scan = new Scan();
332 scan.setMaxResultsPerColumnFamily(4);
333 ResultScanner scanner = ht.getScanner(scan);
334 kvListScan = new ArrayList<Cell>();
335 while ((result = scanner.next()) != null) {
336 for (Cell kv : result.listCells()) {
337 kvListScan.add(kv);
338 }
339 }
340 result = Result.create(kvListScan);
341 verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
342
343 }
344
345
346
347
348
349
350 @Test
351 public void testGetRowOffset() throws Exception {
352 byte [] TABLE = Bytes.toBytes("testGetRowOffset");
353 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
354 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
355
356 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
357
358 Get get;
359 Put put;
360 Result result;
361 boolean toLog = true;
362 List<Cell> kvListExp;
363
364
365 kvListExp = new ArrayList<Cell>();
366 put = new Put(ROW);
367 for (int i=0; i < 10; i++) {
368 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
369 put.add(kv);
370
371 if (i < 2) continue;
372 kvListExp.add(kv);
373 }
374 ht.put(put);
375
376
377 get = new Get(ROW);
378 get.setRowOffsetPerColumnFamily(2);
379 result = ht.get(get);
380 verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
381
382
383 get = new Get(ROW);
384 get.setRowOffsetPerColumnFamily(20);
385 result = ht.get(get);
386 kvListExp = new ArrayList<Cell>();
387 verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
388
389
390 get = new Get(ROW);
391 get.setRowOffsetPerColumnFamily(4);
392 get.setMaxResultsPerColumnFamily(5);
393 result = ht.get(get);
394 kvListExp = new ArrayList<Cell>();
395 for (int i=4; i < 9; i++) {
396 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
397 }
398 verifyResult(result, kvListExp, toLog,
399 "Testing offset + setMaxResultsPerCF");
400
401
402 get = new Get(ROW);
403 get.setRowOffsetPerColumnFamily(1);
404 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
405 true));
406 result = ht.get(get);
407 kvListExp = new ArrayList<Cell>();
408 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
409 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
410 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
411 verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
412
413
414
415 for(int j=2; j > 0; j--) {
416 put = new Put(ROW);
417 for (int i=0; i < 10; i++) {
418 KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
419 put.add(kv);
420 }
421 ht.put(put);
422 }
423
424 get = new Get(ROW);
425 get.setRowOffsetPerColumnFamily(4);
426 get.setMaxResultsPerColumnFamily(2);
427 get.addFamily(FAMILIES[1]);
428 get.addFamily(FAMILIES[2]);
429 result = ht.get(get);
430 kvListExp = new ArrayList<Cell>();
431
432 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
433 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
434 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
435 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
436 verifyResult(result, kvListExp, toLog,
437 "Testing offset + multiple CFs + maxResults");
438 }
439
440
441
442
443
444
445
446 @Test
447 public void testScanOnReopenedRegion() throws Exception {
448 TableName TABLE = TableName.valueOf("testScanOnReopenedRegion");
449 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 2);
450
451 HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
452
453 Put put;
454 Scan scan;
455 Result result;
456 ResultScanner scanner;
457 boolean toLog = false;
458 List<Cell> kvListExp;
459
460
461 put = new Put(ROW);
462 for (int i=0; i < QUALIFIERS.length; i++) {
463 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
464 put.add(kv);
465 }
466 ht.put(put);
467
468 scan = new Scan(ROW);
469 scanner = ht.getScanner(scan);
470
471 HRegionLocation loc = ht.getRegionLocation(ROW);
472 HRegionInfo hri = loc.getRegionInfo();
473 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
474 byte[] regionName = hri.getRegionName();
475 int i = cluster.getServerWith(regionName);
476 HRegionServer rs = cluster.getRegionServer(i);
477 ProtobufUtil.closeRegion(
478 rs.getRSRpcServices(), rs.getServerName(), regionName, false);
479 long startTime = EnvironmentEdgeManager.currentTime();
480 long timeOut = 300000;
481 while (true) {
482 if (rs.getOnlineRegion(regionName) == null) {
483 break;
484 }
485 assertTrue("Timed out in closing the testing region",
486 EnvironmentEdgeManager.currentTime() < startTime + timeOut);
487 Thread.sleep(500);
488 }
489
490
491 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
492 try {
493 HMaster master = cluster.getMaster();
494 RegionStates states = master.getAssignmentManager().getRegionStates();
495 states.regionOffline(hri);
496 states.updateRegionState(hri, State.OPENING);
497 if (ConfigUtil.useZKForAssignment(TEST_UTIL.getConfiguration())) {
498 ZKAssign.createNodeOffline(zkw, hri, loc.getServerName());
499 }
500 ProtobufUtil.openRegion(rs.getRSRpcServices(), rs.getServerName(), hri);
501 startTime = EnvironmentEdgeManager.currentTime();
502 while (true) {
503 if (rs.getOnlineRegion(regionName) != null) {
504 break;
505 }
506 assertTrue("Timed out in open the testing region",
507 EnvironmentEdgeManager.currentTime() < startTime + timeOut);
508 Thread.sleep(500);
509 }
510 } finally {
511 ZKAssign.deleteNodeFailSilent(zkw, hri);
512 }
513
514
515 kvListExp = new ArrayList<Cell>();
516 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[0], 0, VALUE));
517 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[1], 1, VALUE));
518 result = scanner.next();
519 verifyResult(result, kvListExp, toLog, "Testing scan on re-opened region");
520 }
521
522 static void verifyResult(Result result, List<Cell> expKvList, boolean toLog,
523 String msg) {
524
525 LOG.info(msg);
526 LOG.info("Expected count: " + expKvList.size());
527 LOG.info("Actual count: " + result.size());
528 if (expKvList.size() == 0)
529 return;
530
531 int i = 0;
532 for (Cell kv : result.rawCells()) {
533 if (i >= expKvList.size()) {
534 break;
535 }
536
537 Cell kvExp = expKvList.get(i++);
538 if (toLog) {
539 LOG.info("get kv is: " + kv.toString());
540 LOG.info("exp kv is: " + kvExp.toString());
541 }
542 assertTrue("Not equal", kvExp.equals(kv));
543 }
544
545 assertEquals(expKvList.size(), result.size());
546 }
547
548
549 }