1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.regionserver;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.Random;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.HBaseTestCase;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Scan;
38  import org.apache.hadoop.hbase.io.hfile.Compression;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hdfs.MiniDFSCluster;
41  
42  public class TestWideScanner extends HBaseTestCase {
43    private final Log LOG = LogFactory.getLog(this.getClass());
44  
45    static final byte[] A = Bytes.toBytes("A");
46    static final byte[] B = Bytes.toBytes("B");
47    static final byte[] C = Bytes.toBytes("C");
48    static byte[][] COLUMNS = { A, B, C };
49    static final Random rng = new Random();
50    static final HTableDescriptor TESTTABLEDESC =
51      new HTableDescriptor("testwidescan");
52    static {
53      TESTTABLEDESC.addFamily(new HColumnDescriptor(A,
54        10,  // Ten is arbitrary number.  Keep versions to help debuggging.
55        Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
56        HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
57        HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
58      TESTTABLEDESC.addFamily(new HColumnDescriptor(B,
59        10,  // Ten is arbitrary number.  Keep versions to help debuggging.
60        Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
61        HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
62        HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
63      TESTTABLEDESC.addFamily(new HColumnDescriptor(C,
64        10,  // Ten is arbitrary number.  Keep versions to help debuggging.
65        Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
66        HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
67        HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
68    }
69  
70    /** HRegionInfo for root region */
71    public static final HRegionInfo REGION_INFO =
72      new HRegionInfo(TESTTABLEDESC, HConstants.EMPTY_BYTE_ARRAY,
73      HConstants.EMPTY_BYTE_ARRAY);
74  
75    MiniDFSCluster cluster = null;
76    HRegion r;
77  
78    @Override
79    public void setUp() throws Exception {
80      cluster = new MiniDFSCluster(conf, 2, true, (String[])null);
81      // Set the hbase.rootdir to be the home directory in mini dfs.
82      this.conf.set(HConstants.HBASE_DIR,
83        this.cluster.getFileSystem().getHomeDirectory().toString());
84      super.setUp();
85    }
86  
87    private int addWideContent(HRegion region) throws IOException {
88      int count = 0;
89      for (char c = 'a'; c <= 'c'; c++) {
90        byte[] row = Bytes.toBytes("ab" + c);
91        int i;
92        for (i = 0; i < 2500; i++) {
93          byte[] b = Bytes.toBytes(String.format("%10d", i));
94          Put put = new Put(row);
95          put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, b);
96          region.put(put);
97          count++;
98        }
99      }
100     return count;
101   }
102 
103   public void testWideScanBatching() throws IOException {
104     try {
105       this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
106       int inserted = addWideContent(this.r);
107       List<KeyValue> results = new ArrayList<KeyValue>();
108       Scan scan = new Scan();
109       scan.addFamily(A);
110       scan.addFamily(B);
111       scan.addFamily(C);
112       scan.setBatch(1000);
113       InternalScanner s = r.getScanner(scan);
114       int total = 0;
115       int i = 0;
116       boolean more;
117       do {
118         more = s.next(results);
119         i++;
120         LOG.info("iteration #" + i + ", results.size=" + results.size());
121 
122         // assert that the result set is no larger than 1000
123         assertTrue(results.size() <= 1000);
124 
125         total += results.size();
126 
127         if (results.size() > 0) {
128           // assert that all results are from the same row
129           byte[] row = results.get(0).getRow();
130           for (KeyValue kv: results) {
131             assertTrue(Bytes.equals(row, kv.getRow()));
132           }
133         }
134 
135         results.clear();
136       } while (more);
137 
138       // assert that the scanner returned all values
139       LOG.info("inserted " + inserted + ", scanned " + total);
140       assertTrue(total == inserted);
141 
142       s.close();
143     } finally {
144       this.r.close();
145       this.r.getLog().closeAndDelete();
146       shutdownDfs(this.cluster);
147     }
148   }
149 }