1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver;
22
23 import static org.junit.Assert.*;
24
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.Collection;
28 import java.util.HashMap;
29 import java.util.HashSet;
30 import java.util.List;
31 import java.util.Set;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.hbase.*;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Scan;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41
42 @Category(SmallTests.class)
43 public class TestColumnSeeking {
44
45 private final static HBaseTestingUtility TEST_UTIL =
46 new HBaseTestingUtility();
47
48 static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
49
50 @SuppressWarnings("unchecked")
51 @Test
52 public void testDuplicateVersions() throws IOException {
53 String family = "Family";
54 byte[] familyBytes = Bytes.toBytes("Family");
55 String table = "TestDuplicateVersions";
56
57 HColumnDescriptor hcd =
58 new HColumnDescriptor(familyBytes).setMaxVersions(1000);
59 HTableDescriptor htd = new HTableDescriptor(table);
60 htd.addFamily(hcd);
61 HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false);
62 HRegion region =
63 HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
64 .getConfiguration(), htd);
65 try {
66 List<String> rows = generateRandomWords(10, "row");
67 List<String> allColumns = generateRandomWords(10, "column");
68 List<String> values = generateRandomWords(100, "value");
69
70 long maxTimestamp = 2;
71 double selectPercent = 0.5;
72 int numberOfTests = 5;
73 double flushPercentage = 0.2;
74 double minorPercentage = 0.2;
75 double majorPercentage = 0.2;
76 double putPercentage = 0.2;
77
78 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
79
80 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
81 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
82
83 for (int i = 0; i < numberOfTests; i++) {
84 kvMaps[i] = new HashMap<String, KeyValue>();
85 columnLists[i] = new ArrayList<String>();
86 for (String column : allColumns) {
87 if (Math.random() < selectPercent) {
88 columnLists[i].add(column);
89 }
90 }
91 }
92
93 for (String value : values) {
94 for (String row : rows) {
95 Put p = new Put(Bytes.toBytes(row));
96 p.setWriteToWAL(false);
97 for (String column : allColumns) {
98 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
99 KeyValue kv =
100 KeyValueTestUtil.create(row, family, column, timestamp, value);
101 if (Math.random() < putPercentage) {
102 p.add(kv);
103 allKVMap.put(kv.getKeyString(), kv);
104 for (int i = 0; i < numberOfTests; i++) {
105 if (columnLists[i].contains(column)) {
106 kvMaps[i].put(kv.getKeyString(), kv);
107 }
108 }
109 }
110 }
111 }
112 region.put(p);
113 if (Math.random() < flushPercentage) {
114 LOG.info("Flushing... ");
115 region.flushcache();
116 }
117
118 if (Math.random() < minorPercentage) {
119 LOG.info("Minor compacting... ");
120 region.compactStores(false);
121 }
122
123 if (Math.random() < majorPercentage) {
124 LOG.info("Major compacting... ");
125 region.compactStores(true);
126 }
127 }
128 }
129
130 for (int i = 0; i < numberOfTests + 1; i++) {
131 Collection<KeyValue> kvSet;
132 Scan scan = new Scan();
133 scan.setMaxVersions();
134 if (i < numberOfTests) {
135 if (columnLists[i].size() == 0) continue;
136 kvSet = kvMaps[i].values();
137 for (String column : columnLists[i]) {
138 scan.addColumn(familyBytes, Bytes.toBytes(column));
139 }
140 LOG.info("ExplicitColumns scanner");
141 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
142 + kvSet.size());
143 } else {
144 kvSet = allKVMap.values();
145 LOG.info("Wildcard scanner");
146 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
147
148 }
149 InternalScanner scanner = region.getScanner(scan);
150 List<KeyValue> results = new ArrayList<KeyValue>();
151 while (scanner.next(results))
152 ;
153 assertEquals(kvSet.size(), results.size());
154 assertTrue(results.containsAll(kvSet));
155 }
156 } finally {
157 HRegion.closeHRegion(region);
158 }
159
160 region.close();
161 region.getLog().closeAndDelete();
162 }
163
164 @SuppressWarnings("unchecked")
165 @Test
166 public void testReseeking() throws IOException {
167 String family = "Family";
168 byte[] familyBytes = Bytes.toBytes("Family");
169 String table = "TestSingleVersions";
170
171 HTableDescriptor htd = new HTableDescriptor(table);
172 htd.addFamily(new HColumnDescriptor(family));
173
174 HRegionInfo info = new HRegionInfo(Bytes.toBytes(table), null, null, false);
175 HRegion region =
176 HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
177 .getConfiguration(), htd);
178
179 List<String> rows = generateRandomWords(10, "row");
180 List<String> allColumns = generateRandomWords(100, "column");
181
182 long maxTimestamp = 2;
183 double selectPercent = 0.5;
184 int numberOfTests = 5;
185 double flushPercentage = 0.2;
186 double minorPercentage = 0.2;
187 double majorPercentage = 0.2;
188 double putPercentage = 0.2;
189
190 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
191
192 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
193 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
194 String valueString = "Value";
195
196 for (int i = 0; i < numberOfTests; i++) {
197 kvMaps[i] = new HashMap<String, KeyValue>();
198 columnLists[i] = new ArrayList<String>();
199 for (String column : allColumns) {
200 if (Math.random() < selectPercent) {
201 columnLists[i].add(column);
202 }
203 }
204 }
205
206 for (String row : rows) {
207 Put p = new Put(Bytes.toBytes(row));
208 p.setWriteToWAL(false);
209 for (String column : allColumns) {
210 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
211 KeyValue kv =
212 KeyValueTestUtil.create(row, family, column, timestamp,
213 valueString);
214 if (Math.random() < putPercentage) {
215 p.add(kv);
216 allKVMap.put(kv.getKeyString(), kv);
217 for (int i = 0; i < numberOfTests; i++) {
218 if (columnLists[i].contains(column)) {
219 kvMaps[i].put(kv.getKeyString(), kv);
220 }
221 }
222 }
223
224 }
225 }
226 region.put(p);
227 if (Math.random() < flushPercentage) {
228 LOG.info("Flushing... ");
229 region.flushcache();
230 }
231
232 if (Math.random() < minorPercentage) {
233 LOG.info("Minor compacting... ");
234 region.compactStores(false);
235 }
236
237 if (Math.random() < majorPercentage) {
238 LOG.info("Major compacting... ");
239 region.compactStores(true);
240 }
241 }
242
243 for (int i = 0; i < numberOfTests + 1; i++) {
244 Collection<KeyValue> kvSet;
245 Scan scan = new Scan();
246 scan.setMaxVersions();
247 if (i < numberOfTests) {
248 if (columnLists[i].size() == 0) continue;
249 kvSet = kvMaps[i].values();
250 for (String column : columnLists[i]) {
251 scan.addColumn(familyBytes, Bytes.toBytes(column));
252 }
253 LOG.info("ExplicitColumns scanner");
254 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
255 + kvSet.size());
256 } else {
257 kvSet = allKVMap.values();
258 LOG.info("Wildcard scanner");
259 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
260
261 }
262 InternalScanner scanner = region.getScanner(scan);
263 List<KeyValue> results = new ArrayList<KeyValue>();
264 while (scanner.next(results))
265 ;
266 assertEquals(kvSet.size(), results.size());
267 assertTrue(results.containsAll(kvSet));
268 }
269
270 region.close();
271 region.getLog().closeAndDelete();
272 }
273
274 List<String> generateRandomWords(int numberOfWords, String suffix) {
275 Set<String> wordSet = new HashSet<String>();
276 for (int i = 0; i < numberOfWords; i++) {
277 int lengthOfWords = (int) (Math.random() * 5) + 1;
278 char[] wordChar = new char[lengthOfWords];
279 for (int j = 0; j < wordChar.length; j++) {
280 wordChar[j] = (char) (Math.random() * 26 + 97);
281 }
282 String word;
283 if (suffix == null) {
284 word = new String(wordChar);
285 } else {
286 word = new String(wordChar) + suffix;
287 }
288 wordSet.add(word);
289 }
290 List<String> wordList = new ArrayList<String>(wordSet);
291 return wordList;
292 }
293
294 @org.junit.Rule
295 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
296 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
297 }
298