View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.client;
18  
19  import static org.junit.Assert.assertEquals;
20  import static org.junit.Assert.assertTrue;
21  
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.HBaseTestingUtility;
28  import org.apache.hadoop.hbase.HTestConst;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.MediumTests;
31  import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
32  import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.junit.After;
35  import org.junit.AfterClass;
36  import org.junit.Before;
37  import org.junit.BeforeClass;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  
41  /**
42   * A client-side test, mostly testing scanners with various parameters.
43   */
44  @Category(MediumTests.class)
45  public class TestScannersFromClientSide {
46    private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
47  
48    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49    private static byte [] ROW = Bytes.toBytes("testRow");
50    private static byte [] FAMILY = Bytes.toBytes("testFamily");
51    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
52    private static byte [] VALUE = Bytes.toBytes("testValue");
53  
54    /**
55     * @throws java.lang.Exception
56     */
57    @BeforeClass
58    public static void setUpBeforeClass() throws Exception {
59      TEST_UTIL.startMiniCluster(3);
60    }
61  
62    /**
63     * @throws java.lang.Exception
64     */
65    @AfterClass
66    public static void tearDownAfterClass() throws Exception {
67      TEST_UTIL.shutdownMiniCluster();
68    }
69  
70    /**
71     * @throws java.lang.Exception
72     */
73    @Before
74    public void setUp() throws Exception {
75      // Nothing to do.
76    }
77  
78    /**
79     * @throws java.lang.Exception
80     */
81    @After
82    public void tearDown() throws Exception {
83      // Nothing to do.
84    }
85  
86    /**
87     * Test from client side for batch of scan
88     *
89     * @throws Exception
90     */
91    @Test
92    public void testScanBatch() throws Exception {
93      byte [] TABLE = Bytes.toBytes("testScanBatch");
94      byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
95  
96      HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
97  
98      Put put;
99      Scan scan;
100     Delete delete;
101     Result result;
102     ResultScanner scanner;
103     boolean toLog = true;
104     List<KeyValue> kvListExp;
105 
106     // table: row, family, c0:0, c1:1, ... , c7:7
107     put = new Put(ROW);
108     for (int i=0; i < QUALIFIERS.length; i++) {
109       KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
110       put.add(kv);
111     }
112     ht.put(put);
113 
114     // table: row, family, c0:0, c1:1, ..., c6:2, c6:6 , c7:7
115     put = new Put(ROW);
116     KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
117     put.add(kv);
118     ht.put(put);
119 
120     // delete upto ts: 3
121     delete = new Delete(ROW);
122     delete.deleteFamily(FAMILY, 3);
123     ht.delete(delete);
124 
125     // without batch
126     scan = new Scan(ROW);
127     scan.setMaxVersions();
128     scanner = ht.getScanner(scan);
129 
130     // c4:4, c5:5, c6:6, c7:7
131     kvListExp = new ArrayList<KeyValue>();
132     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
133     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
134     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
135     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
136     result = scanner.next();
137     verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
138 
139     // with batch
140     scan = new Scan(ROW);
141     scan.setMaxVersions();
142     scan.setBatch(2);
143     scanner = ht.getScanner(scan);
144 
145     // First batch: c4:4, c5:5
146     kvListExp = new ArrayList<KeyValue>();
147     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
148     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
149     result = scanner.next();
150     verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
151 
152     // Second batch: c6:6, c7:7
153     kvListExp = new ArrayList<KeyValue>();
154     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
155     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
156     result = scanner.next();
157     verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
158 
159   }
160 
161   /**
162    * Test from client side for get with maxResultPerCF set
163    *
164    * @throws Exception
165    */
166   @Test
167   public void testGetMaxResults() throws Exception {
168     byte [] TABLE = Bytes.toBytes("testGetMaxResults");
169     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
170     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
171 
172     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
173 
174     Get get;
175     Put put;
176     Result result;
177     boolean toLog = true;
178     List<KeyValue> kvListExp;
179 
180     kvListExp = new ArrayList<KeyValue>();
181     // Insert one CF for row[0]
182     put = new Put(ROW);
183     for (int i=0; i < 10; i++) {
184       KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
185       put.add(kv);
186       kvListExp.add(kv);
187     }
188     ht.put(put);
189 
190     get = new Get(ROW);
191     result = ht.get(get);
192     verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
193 
194     get = new Get(ROW);
195     get.setMaxResultsPerColumnFamily(2);
196     result = ht.get(get);
197     kvListExp = new ArrayList<KeyValue>();
198     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
199     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
200     verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
201 
202     // Filters: ColumnRangeFilter
203     get = new Get(ROW);
204     get.setMaxResultsPerColumnFamily(5);
205     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
206                                         true));
207     result = ht.get(get);
208     kvListExp = new ArrayList<KeyValue>();
209     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
210     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
211     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
212     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
213     verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
214 
215     // Insert two more CF for row[0]
216     // 20 columns for CF2, 10 columns for CF1
217     put = new Put(ROW);
218     for (int i=0; i < QUALIFIERS.length; i++) {
219       KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
220       put.add(kv);
221     }
222     ht.put(put);
223 
224     put = new Put(ROW);
225     for (int i=0; i < 10; i++) {
226       KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
227       put.add(kv);
228     }
229     ht.put(put);
230 
231     get = new Get(ROW);
232     get.setMaxResultsPerColumnFamily(12);
233     get.addFamily(FAMILIES[1]);
234     get.addFamily(FAMILIES[2]);
235     result = ht.get(get);
236     kvListExp = new ArrayList<KeyValue>();
237     //Exp: CF1:q0, ..., q9, CF2: q0, q1, q10, q11, ..., q19
238     for (int i=0; i < 10; i++) {
239       kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
240     }
241     for (int i=0; i < 2; i++) {
242         kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
243       }
244     for (int i=10; i < 20; i++) {
245       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
246     }
247     verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
248 
249     // Filters: ColumnRangeFilter and ColumnPrefixFilter
250     get = new Get(ROW);
251     get.setMaxResultsPerColumnFamily(3);
252     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
253     result = ht.get(get);
254     kvListExp = new ArrayList<KeyValue>();
255     for (int i=2; i < 5; i++) {
256       kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
257     }
258     for (int i=2; i < 5; i++) {
259       kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
260     }
261     for (int i=2; i < 5; i++) {
262       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
263     }
264     verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
265 
266     get = new Get(ROW);
267     get.setMaxResultsPerColumnFamily(7);
268     get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
269     result = ht.get(get);
270     kvListExp = new ArrayList<KeyValue>();
271     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
272     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
273     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
274     for (int i=10; i < 16; i++) {
275       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
276     }
277     verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
278 
279   }
280 
281   /**
282    * Test from client side for scan with maxResultPerCF set
283    *
284    * @throws Exception
285    */
286   @Test
287   public void testScanMaxResults() throws Exception {
288     byte [] TABLE = Bytes.toBytes("testScanLimit");
289     byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
290     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
291     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
292 
293     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
294 
295     Put put;
296     Scan scan;
297     Result result;
298     boolean toLog = true;
299     List<KeyValue> kvListExp, kvListScan;
300 
301     kvListExp = new ArrayList<KeyValue>();
302 
303     for (int r=0; r < ROWS.length; r++) {
304       put = new Put(ROWS[r]);
305       for (int c=0; c < FAMILIES.length; c++) {
306         for (int q=0; q < QUALIFIERS.length; q++) {
307           KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
308           put.add(kv);
309           if (q < 4) {
310             kvListExp.add(kv);
311           }
312         }
313       }
314       ht.put(put);
315     }
316 
317     scan = new Scan();
318     scan.setMaxResultsPerColumnFamily(4);
319     ResultScanner scanner = ht.getScanner(scan);
320     kvListScan = new ArrayList<KeyValue>();
321     while ((result = scanner.next()) != null) {
322       for (KeyValue kv : result.list()) {
323         kvListScan.add(kv);
324       }
325     }
326     result = new Result(kvListScan);
327     verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
328 
329   }
330 
331   /**
332    * Test from client side for get with rowOffset
333    *
334    * @throws Exception
335    */
336   @Test
337   public void testGetRowOffset() throws Exception {
338     byte [] TABLE = Bytes.toBytes("testGetRowOffset");
339     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
340     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
341 
342     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
343 
344     Get get;
345     Put put;
346     Result result;
347     boolean toLog = true;
348     List<KeyValue> kvListExp;
349 
350     // Insert one CF for row
351     kvListExp = new ArrayList<KeyValue>();
352     put = new Put(ROW);
353     for (int i=0; i < 10; i++) {
354       KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
355       put.add(kv);
356       // skipping first two kvs
357       if (i < 2) continue;
358       kvListExp.add(kv);
359     }
360     ht.put(put);
361 
362     //setting offset to 2
363     get = new Get(ROW);
364     get.setRowOffsetPerColumnFamily(2);
365     result = ht.get(get);
366     verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
367 
368     //setting offset to 20
369     get = new Get(ROW);
370     get.setRowOffsetPerColumnFamily(20);
371     result = ht.get(get);
372     kvListExp = new ArrayList<KeyValue>();
373     verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
374 
375     //offset + maxResultPerCF
376     get = new Get(ROW);
377     get.setRowOffsetPerColumnFamily(4);
378     get.setMaxResultsPerColumnFamily(5);
379     result = ht.get(get);
380     kvListExp = new ArrayList<KeyValue>();
381     for (int i=4; i < 9; i++) {
382       kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
383     }
384     verifyResult(result, kvListExp, toLog,
385       "Testing offset + setMaxResultsPerCF");
386 
387     // Filters: ColumnRangeFilter
388     get = new Get(ROW);
389     get.setRowOffsetPerColumnFamily(1);
390     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
391                                         true));
392     result = ht.get(get);
393     kvListExp = new ArrayList<KeyValue>();
394     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
395     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
396     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
397     verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
398 
399     // Insert into two more CFs for row
400     // 10 columns for CF2, 10 columns for CF1
401     for(int j=2; j > 0; j--) {
402       put = new Put(ROW);
403       for (int i=0; i < 10; i++) {
404         KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
405         put.add(kv);
406       }
407       ht.put(put);
408     }
409 
410     get = new Get(ROW);
411     get.setRowOffsetPerColumnFamily(4);
412     get.setMaxResultsPerColumnFamily(2);
413     get.addFamily(FAMILIES[1]);
414     get.addFamily(FAMILIES[2]);
415     result = ht.get(get);
416     kvListExp = new ArrayList<KeyValue>();
417     //Exp: CF1:q4, q5, CF2: q4, q5
418     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
419     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
420     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
421     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
422     verifyResult(result, kvListExp, toLog,
423        "Testing offset + multiple CFs + maxResults");
424   }
425 
426   static void verifyResult(Result result, List<KeyValue> expKvList, boolean toLog,
427       String msg) {
428 
429     LOG.info(msg);
430     LOG.info("Expected count: " + expKvList.size());
431     LOG.info("Actual count: " + result.size());
432     if (expKvList.size() == 0)
433       return;
434 
435     int i = 0;
436     for (KeyValue kv : result.raw()) {
437       if (i >= expKvList.size()) {
438         break;  // we will check the size later
439       }
440 
441       KeyValue kvExp = expKvList.get(i++);
442       if (toLog) {
443         LOG.info("get kv is: " + kv.toString());
444         LOG.info("exp kv is: " + kvExp.toString());
445       }
446       assertTrue("Not equal", kvExp.equals(kv));
447     }
448 
449     assertEquals(expKvList.size(), result.size());
450   }
451 
452 
453 }