1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.regionserver;
22  
23  import static org.junit.Assert.*;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.Collection;
28  import java.util.HashMap;
29  import java.util.HashSet;
30  import java.util.List;
31  import java.util.Set;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.hbase.*;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Scan;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  
42  @Category(SmallTests.class)
43  public class TestColumnSeeking {
44  
45    private final static HBaseTestingUtility TEST_UTIL =
46        new HBaseTestingUtility();
47  
48    static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
49  
50    @SuppressWarnings("unchecked")
51    @Test
52    public void testDuplicateVersions() throws IOException {
53      String family = "Family";
54      byte[] familyBytes = Bytes.toBytes("Family");
55      String table = "TestDuplicateVersions";
56  
57      HColumnDescriptor hcd =
58          new HColumnDescriptor(familyBytes).setMaxVersions(1000);
59      HTableDescriptor htd = new HTableDescriptor(table);
60      htd.addFamily(hcd);
61      HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false);
62      HRegion region =
63          HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
64              .getConfiguration(), htd);
65      try {
66        List<String> rows = generateRandomWords(10, "row");
67        List<String> allColumns = generateRandomWords(10, "column");
68        List<String> values = generateRandomWords(100, "value");
69  
70        long maxTimestamp = 2;
71        double selectPercent = 0.5;
72        int numberOfTests = 5;
73        double flushPercentage = 0.2;
74        double minorPercentage = 0.2;
75        double majorPercentage = 0.2;
76        double putPercentage = 0.2;
77  
78        HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
79  
80        HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
81        ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
82  
83        for (int i = 0; i < numberOfTests; i++) {
84          kvMaps[i] = new HashMap<String, KeyValue>();
85          columnLists[i] = new ArrayList<String>();
86          for (String column : allColumns) {
87            if (Math.random() < selectPercent) {
88              columnLists[i].add(column);
89            }
90          }
91        }
92  
93        for (String value : values) {
94          for (String row : rows) {
95            Put p = new Put(Bytes.toBytes(row));
96            p.setWriteToWAL(false);
97            for (String column : allColumns) {
98              for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
99                KeyValue kv =
100                   KeyValueTestUtil.create(row, family, column, timestamp, value);
101               if (Math.random() < putPercentage) {
102                 p.add(kv);
103                 allKVMap.put(kv.getKeyString(), kv);
104                 for (int i = 0; i < numberOfTests; i++) {
105                   if (columnLists[i].contains(column)) {
106                     kvMaps[i].put(kv.getKeyString(), kv);
107                   }
108                 }
109               }
110             }
111           }
112           region.put(p);
113           if (Math.random() < flushPercentage) {
114             LOG.info("Flushing... ");
115             region.flushcache();
116           }
117 
118           if (Math.random() < minorPercentage) {
119             LOG.info("Minor compacting... ");
120             region.compactStores(false);
121           }
122 
123           if (Math.random() < majorPercentage) {
124             LOG.info("Major compacting... ");
125             region.compactStores(true);
126           }
127         }
128       }
129 
130       for (int i = 0; i < numberOfTests + 1; i++) {
131         Collection<KeyValue> kvSet;
132         Scan scan = new Scan();
133         scan.setMaxVersions();
134         if (i < numberOfTests) {
135           if (columnLists[i].size() == 0) continue; // HBASE-7700
136           kvSet = kvMaps[i].values();
137           for (String column : columnLists[i]) {
138             scan.addColumn(familyBytes, Bytes.toBytes(column));
139           }
140           LOG.info("ExplicitColumns scanner");
141           LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
142               + kvSet.size());
143         } else {
144           kvSet = allKVMap.values();
145           LOG.info("Wildcard scanner");
146           LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
147 
148         }
149         InternalScanner scanner = region.getScanner(scan);
150         List<KeyValue> results = new ArrayList<KeyValue>();
151         while (scanner.next(results))
152           ;
153         assertEquals(kvSet.size(), results.size());
154         assertTrue(results.containsAll(kvSet));
155       }
156     } finally {
157       HRegion.closeHRegion(region);
158     }
159 
160     region.close();
161     region.getLog().closeAndDelete();
162   }
163 
164   @SuppressWarnings("unchecked")
165   @Test
166   public void testReseeking() throws IOException {
167     String family = "Family";
168     byte[] familyBytes = Bytes.toBytes("Family");
169     String table = "TestSingleVersions";
170 
171     HTableDescriptor htd = new HTableDescriptor(table);
172     htd.addFamily(new HColumnDescriptor(family));
173 
174     HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false);
175     HRegion region =
176         HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
177             .getConfiguration(), htd);
178 
179     List<String> rows = generateRandomWords(10, "row");
180     List<String> allColumns = generateRandomWords(100, "column");
181 
182     long maxTimestamp = 2;
183     double selectPercent = 0.5;
184     int numberOfTests = 5;
185     double flushPercentage = 0.2;
186     double minorPercentage = 0.2;
187     double majorPercentage = 0.2;
188     double putPercentage = 0.2;
189 
190     HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
191 
192     HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
193     ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
194     String valueString = "Value";
195 
196     for (int i = 0; i < numberOfTests; i++) {
197       kvMaps[i] = new HashMap<String, KeyValue>();
198       columnLists[i] = new ArrayList<String>();
199       for (String column : allColumns) {
200         if (Math.random() < selectPercent) {
201           columnLists[i].add(column);
202         }
203       }
204     }
205 
206     for (String row : rows) {
207       Put p = new Put(Bytes.toBytes(row));
208       p.setWriteToWAL(false);
209       for (String column : allColumns) {
210         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
211           KeyValue kv =
212               KeyValueTestUtil.create(row, family, column, timestamp,
213                   valueString);
214           if (Math.random() < putPercentage) {
215             p.add(kv);
216             allKVMap.put(kv.getKeyString(), kv);
217             for (int i = 0; i < numberOfTests; i++) {
218               if (columnLists[i].contains(column)) {
219                 kvMaps[i].put(kv.getKeyString(), kv);
220               }
221             }
222           }
223 
224         }
225       }
226       region.put(p);
227       if (Math.random() < flushPercentage) {
228         LOG.info("Flushing... ");
229         region.flushcache();
230       }
231 
232       if (Math.random() < minorPercentage) {
233         LOG.info("Minor compacting... ");
234         region.compactStores(false);
235       }
236 
237       if (Math.random() < majorPercentage) {
238         LOG.info("Major compacting... ");
239         region.compactStores(true);
240       }
241     }
242 
243     for (int i = 0; i < numberOfTests + 1; i++) {
244       Collection<KeyValue> kvSet;
245       Scan scan = new Scan();
246       scan.setMaxVersions();
247       if (i < numberOfTests) {
248         if (columnLists[i].size() == 0) continue; // HBASE-7700
249         kvSet = kvMaps[i].values();
250         for (String column : columnLists[i]) {
251           scan.addColumn(familyBytes, Bytes.toBytes(column));
252         }
253         LOG.info("ExplicitColumns scanner");
254         LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
255             + kvSet.size());
256       } else {
257         kvSet = allKVMap.values();
258         LOG.info("Wildcard scanner");
259         LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
260 
261       }
262       InternalScanner scanner = region.getScanner(scan);
263       List<KeyValue> results = new ArrayList<KeyValue>();
264       while (scanner.next(results))
265         ;
266       assertEquals(kvSet.size(), results.size());
267       assertTrue(results.containsAll(kvSet));
268     }
269 
270     region.close();
271     region.getLog().closeAndDelete();
272   }
273 
274   List<String> generateRandomWords(int numberOfWords, String suffix) {
275     Set<String> wordSet = new HashSet<String>();
276     for (int i = 0; i < numberOfWords; i++) {
277       int lengthOfWords = (int) (Math.random() * 5) + 1;
278       char[] wordChar = new char[lengthOfWords];
279       for (int j = 0; j < wordChar.length; j++) {
280         wordChar[j] = (char) (Math.random() * 26 + 97);
281       }
282       String word;
283       if (suffix == null) {
284         word = new String(wordChar);
285       } else {
286         word = new String(wordChar) + suffix;
287       }
288       wordSet.add(word);
289     }
290     List<String> wordList = new ArrayList<String>(wordSet);
291     return wordList;
292   }
293 
294   @org.junit.Rule
295   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
296     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
297 }
298