View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.client;
18  
19  import static org.junit.Assert.assertEquals;
20  import static org.junit.Assert.assertTrue;
21  
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.Cell;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.HRegionLocation;
31  import org.apache.hadoop.hbase.HTestConst;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.testclassification.MediumTests;
34  import org.apache.hadoop.hbase.MiniHBaseCluster;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
37  import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
38  import org.apache.hadoop.hbase.master.HMaster;
39  import org.apache.hadoop.hbase.master.RegionState.State;
40  import org.apache.hadoop.hbase.master.RegionStates;
41  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
42  import org.apache.hadoop.hbase.regionserver.HRegionServer;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.ConfigUtil;
45  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
46  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
47  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
48  import org.junit.After;
49  import org.junit.AfterClass;
50  import org.junit.Before;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  /**
56   * A client-side test, mostly testing scanners with various parameters.
57   */
58  @Category(MediumTests.class)
59  public class TestScannersFromClientSide {
60    private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
61  
62    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63    private static byte [] ROW = Bytes.toBytes("testRow");
64    private static byte [] FAMILY = Bytes.toBytes("testFamily");
65    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
66    private static byte [] VALUE = Bytes.toBytes("testValue");
67  
68    /**
69     * @throws java.lang.Exception
70     */
71    @BeforeClass
72    public static void setUpBeforeClass() throws Exception {
73      TEST_UTIL.startMiniCluster(3);
74    }
75  
76    /**
77     * @throws java.lang.Exception
78     */
79    @AfterClass
80    public static void tearDownAfterClass() throws Exception {
81      TEST_UTIL.shutdownMiniCluster();
82    }
83  
84    /**
85     * @throws java.lang.Exception
86     */
87    @Before
88    public void setUp() throws Exception {
89      // Nothing to do.
90    }
91  
92    /**
93     * @throws java.lang.Exception
94     */
95    @After
96    public void tearDown() throws Exception {
97      // Nothing to do.
98    }
99  
100   /**
101    * Test from client side for batch of scan
102    *
103    * @throws Exception
104    */
105   @Test
106   public void testScanBatch() throws Exception {
107     TableName TABLE = TableName.valueOf("testScanBatch");
108     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
109 
110     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
111 
112     Put put;
113     Scan scan;
114     Delete delete;
115     Result result;
116     ResultScanner scanner;
117     boolean toLog = true;
118     List<Cell> kvListExp;
119 
120     // table: row, family, c0:0, c1:1, ... , c7:7
121     put = new Put(ROW);
122     for (int i=0; i < QUALIFIERS.length; i++) {
123       KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
124       put.add(kv);
125     }
126     ht.put(put);
127 
128     // table: row, family, c0:0, c1:1, ..., c6:2, c6:6 , c7:7
129     put = new Put(ROW);
130     KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
131     put.add(kv);
132     ht.put(put);
133 
134     // delete upto ts: 3
135     delete = new Delete(ROW);
136     delete.deleteFamily(FAMILY, 3);
137     ht.delete(delete);
138 
139     // without batch
140     scan = new Scan(ROW);
141     scan.setMaxVersions();
142     scanner = ht.getScanner(scan);
143 
144     // c4:4, c5:5, c6:6, c7:7
145     kvListExp = new ArrayList<Cell>();
146     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
147     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
148     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
149     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
150     result = scanner.next();
151     verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
152 
153     // with batch
154     scan = new Scan(ROW);
155     scan.setMaxVersions();
156     scan.setBatch(2);
157     scanner = ht.getScanner(scan);
158 
159     // First batch: c4:4, c5:5
160     kvListExp = new ArrayList<Cell>();
161     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
162     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
163     result = scanner.next();
164     verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
165 
166     // Second batch: c6:6, c7:7
167     kvListExp = new ArrayList<Cell>();
168     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
169     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
170     result = scanner.next();
171     verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
172 
173   }
174 
175   /**
176    * Test from client side for get with maxResultPerCF set
177    *
178    * @throws Exception
179    */
180   @Test
181   public void testGetMaxResults() throws Exception {
182     byte [] TABLE = Bytes.toBytes("testGetMaxResults");
183     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
184     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
185 
186     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
187 
188     Get get;
189     Put put;
190     Result result;
191     boolean toLog = true;
192     List<Cell> kvListExp;
193 
194     kvListExp = new ArrayList<Cell>();
195     // Insert one CF for row[0]
196     put = new Put(ROW);
197     for (int i=0; i < 10; i++) {
198       KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
199       put.add(kv);
200       kvListExp.add(kv);
201     }
202     ht.put(put);
203 
204     get = new Get(ROW);
205     result = ht.get(get);
206     verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
207 
208     get = new Get(ROW);
209     get.setMaxResultsPerColumnFamily(2);
210     result = ht.get(get);
211     kvListExp = new ArrayList<Cell>();
212     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
213     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
214     verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
215 
216     // Filters: ColumnRangeFilter
217     get = new Get(ROW);
218     get.setMaxResultsPerColumnFamily(5);
219     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
220                                         true));
221     result = ht.get(get);
222     kvListExp = new ArrayList<Cell>();
223     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
224     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
225     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
226     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
227     verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
228 
229     // Insert two more CF for row[0]
230     // 20 columns for CF2, 10 columns for CF1
231     put = new Put(ROW);
232     for (int i=0; i < QUALIFIERS.length; i++) {
233       KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
234       put.add(kv);
235     }
236     ht.put(put);
237 
238     put = new Put(ROW);
239     for (int i=0; i < 10; i++) {
240       KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
241       put.add(kv);
242     }
243     ht.put(put);
244 
245     get = new Get(ROW);
246     get.setMaxResultsPerColumnFamily(12);
247     get.addFamily(FAMILIES[1]);
248     get.addFamily(FAMILIES[2]);
249     result = ht.get(get);
250     kvListExp = new ArrayList<Cell>();
251     //Exp: CF1:q0, ..., q9, CF2: q0, q1, q10, q11, ..., q19
252     for (int i=0; i < 10; i++) {
253       kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
254     }
255     for (int i=0; i < 2; i++) {
256         kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
257       }
258     for (int i=10; i < 20; i++) {
259       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
260     }
261     verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
262 
263     // Filters: ColumnRangeFilter and ColumnPrefixFilter
264     get = new Get(ROW);
265     get.setMaxResultsPerColumnFamily(3);
266     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
267     result = ht.get(get);
268     kvListExp = new ArrayList<Cell>();
269     for (int i=2; i < 5; i++) {
270       kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
271     }
272     for (int i=2; i < 5; i++) {
273       kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
274     }
275     for (int i=2; i < 5; i++) {
276       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
277     }
278     verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
279 
280     get = new Get(ROW);
281     get.setMaxResultsPerColumnFamily(7);
282     get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
283     result = ht.get(get);
284     kvListExp = new ArrayList<Cell>();
285     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
286     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
287     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
288     for (int i=10; i < 16; i++) {
289       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
290     }
291     verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
292 
293   }
294 
295   /**
296    * Test from client side for scan with maxResultPerCF set
297    *
298    * @throws Exception
299    */
300   @Test
301   public void testScanMaxResults() throws Exception {
302     byte [] TABLE = Bytes.toBytes("testScanLimit");
303     byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
304     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
305     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
306 
307     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
308 
309     Put put;
310     Scan scan;
311     Result result;
312     boolean toLog = true;
313     List<Cell> kvListExp, kvListScan;
314 
315     kvListExp = new ArrayList<Cell>();
316 
317     for (int r=0; r < ROWS.length; r++) {
318       put = new Put(ROWS[r]);
319       for (int c=0; c < FAMILIES.length; c++) {
320         for (int q=0; q < QUALIFIERS.length; q++) {
321           KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
322           put.add(kv);
323           if (q < 4) {
324             kvListExp.add(kv);
325           }
326         }
327       }
328       ht.put(put);
329     }
330 
331     scan = new Scan();
332     scan.setMaxResultsPerColumnFamily(4);
333     ResultScanner scanner = ht.getScanner(scan);
334     kvListScan = new ArrayList<Cell>();
335     while ((result = scanner.next()) != null) {
336       for (Cell kv : result.listCells()) {
337         kvListScan.add(kv);
338       }
339     }
340     result = Result.create(kvListScan);
341     verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
342 
343   }
344 
345   /**
346    * Test from client side for get with rowOffset
347    *
348    * @throws Exception
349    */
350   @Test
351   public void testGetRowOffset() throws Exception {
352     byte [] TABLE = Bytes.toBytes("testGetRowOffset");
353     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
354     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
355 
356     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
357 
358     Get get;
359     Put put;
360     Result result;
361     boolean toLog = true;
362     List<Cell> kvListExp;
363 
364     // Insert one CF for row
365     kvListExp = new ArrayList<Cell>();
366     put = new Put(ROW);
367     for (int i=0; i < 10; i++) {
368       KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
369       put.add(kv);
370       // skipping first two kvs
371       if (i < 2) continue;
372       kvListExp.add(kv);
373     }
374     ht.put(put);
375 
376     //setting offset to 2
377     get = new Get(ROW);
378     get.setRowOffsetPerColumnFamily(2);
379     result = ht.get(get);
380     verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
381 
382     //setting offset to 20
383     get = new Get(ROW);
384     get.setRowOffsetPerColumnFamily(20);
385     result = ht.get(get);
386     kvListExp = new ArrayList<Cell>();
387     verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
388 
389     //offset + maxResultPerCF
390     get = new Get(ROW);
391     get.setRowOffsetPerColumnFamily(4);
392     get.setMaxResultsPerColumnFamily(5);
393     result = ht.get(get);
394     kvListExp = new ArrayList<Cell>();
395     for (int i=4; i < 9; i++) {
396       kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
397     }
398     verifyResult(result, kvListExp, toLog,
399       "Testing offset + setMaxResultsPerCF");
400 
401     // Filters: ColumnRangeFilter
402     get = new Get(ROW);
403     get.setRowOffsetPerColumnFamily(1);
404     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
405                                         true));
406     result = ht.get(get);
407     kvListExp = new ArrayList<Cell>();
408     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
409     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
410     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
411     verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
412 
413     // Insert into two more CFs for row
414     // 10 columns for CF2, 10 columns for CF1
415     for(int j=2; j > 0; j--) {
416       put = new Put(ROW);
417       for (int i=0; i < 10; i++) {
418         KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
419         put.add(kv);
420       }
421       ht.put(put);
422     }
423 
424     get = new Get(ROW);
425     get.setRowOffsetPerColumnFamily(4);
426     get.setMaxResultsPerColumnFamily(2);
427     get.addFamily(FAMILIES[1]);
428     get.addFamily(FAMILIES[2]);
429     result = ht.get(get);
430     kvListExp = new ArrayList<Cell>();
431     //Exp: CF1:q4, q5, CF2: q4, q5
432     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
433     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
434     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
435     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
436     verifyResult(result, kvListExp, toLog,
437        "Testing offset + multiple CFs + maxResults");
438   }
439 
440   /**
441    * Test from client side for scan while the region is reopened
442    * on the same region server.
443    *
444    * @throws Exception
445    */
446   @Test
447   public void testScanOnReopenedRegion() throws Exception {
448     TableName TABLE = TableName.valueOf("testScanOnReopenedRegion");
449     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 2);
450 
451     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
452 
453     Put put;
454     Scan scan;
455     Result result;
456     ResultScanner scanner;
457     boolean toLog = false;
458     List<Cell> kvListExp;
459 
460     // table: row, family, c0:0, c1:1
461     put = new Put(ROW);
462     for (int i=0; i < QUALIFIERS.length; i++) {
463       KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
464       put.add(kv);
465     }
466     ht.put(put);
467 
468     scan = new Scan(ROW);
469     scanner = ht.getScanner(scan);
470 
471     HRegionLocation loc = ht.getRegionLocation(ROW);
472     HRegionInfo hri = loc.getRegionInfo();
473     MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
474     byte[] regionName = hri.getRegionName();
475     int i = cluster.getServerWith(regionName);
476     HRegionServer rs = cluster.getRegionServer(i);
477     ProtobufUtil.closeRegion(
478       rs.getRSRpcServices(), rs.getServerName(), regionName, false);
479     long startTime = EnvironmentEdgeManager.currentTime();
480     long timeOut = 300000;
481     while (true) {
482       if (rs.getOnlineRegion(regionName) == null) {
483         break;
484       }
485       assertTrue("Timed out in closing the testing region",
486         EnvironmentEdgeManager.currentTime() < startTime + timeOut);
487       Thread.sleep(500);
488     }
489 
490     // Now open the region again.
491     ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
492     try {
493       HMaster master = cluster.getMaster();
494       RegionStates states = master.getAssignmentManager().getRegionStates();
495       states.regionOffline(hri);
496       states.updateRegionState(hri, State.OPENING);
497       if (ConfigUtil.useZKForAssignment(TEST_UTIL.getConfiguration())) {
498         ZKAssign.createNodeOffline(zkw, hri, loc.getServerName());
499       }
500       ProtobufUtil.openRegion(rs.getRSRpcServices(), rs.getServerName(), hri);
501       startTime = EnvironmentEdgeManager.currentTime();
502       while (true) {
503         if (rs.getOnlineRegion(regionName) != null) {
504           break;
505         }
506         assertTrue("Timed out in open the testing region",
507           EnvironmentEdgeManager.currentTime() < startTime + timeOut);
508         Thread.sleep(500);
509       }
510     } finally {
511       ZKAssign.deleteNodeFailSilent(zkw, hri);
512     }
513 
514     // c0:0, c1:1
515     kvListExp = new ArrayList<Cell>();
516     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[0], 0, VALUE));
517     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[1], 1, VALUE));
518     result = scanner.next();
519     verifyResult(result, kvListExp, toLog, "Testing scan on re-opened region");
520   }
521 
522   static void verifyResult(Result result, List<Cell> expKvList, boolean toLog,
523       String msg) {
524 
525     LOG.info(msg);
526     LOG.info("Expected count: " + expKvList.size());
527     LOG.info("Actual count: " + result.size());
528     if (expKvList.size() == 0)
529       return;
530 
531     int i = 0;
532     for (Cell kv : result.rawCells()) {
533       if (i >= expKvList.size()) {
534         break;  // we will check the size later
535       }
536 
537       Cell kvExp = expKvList.get(i++);
538       if (toLog) {
539         LOG.info("get kv is: " + kv.toString());
540         LOG.info("exp kv is: " + kvExp.toString());
541       }
542       assertTrue("Not equal", kvExp.equals(kv));
543     }
544 
545     assertEquals(expKvList.size(), result.size());
546   }
547 
548 
549 }