1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import static org.junit.Assert.*;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Set;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.*;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.client.Durability;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41
42 @Category(SmallTests.class)
43 public class TestColumnSeeking {
44
45 private final static HBaseTestingUtility TEST_UTIL =
46 new HBaseTestingUtility();
47
48 static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
49
50 @SuppressWarnings("unchecked")
51 @Test
52 public void testDuplicateVersions() throws IOException {
53 String family = "Family";
54 byte[] familyBytes = Bytes.toBytes("Family");
55 TableName table = TableName.valueOf("TestDuplicateVersions");
56
57 HColumnDescriptor hcd =
58 new HColumnDescriptor(familyBytes).setMaxVersions(1000);
59 hcd.setMaxVersions(3);
60 HTableDescriptor htd = new HTableDescriptor(table);
61 htd.addFamily(hcd);
62 HRegionInfo info = new HRegionInfo(table, null, null, false);
63 HRegion region =
64 HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
65 .getConfiguration(), htd);
66 try {
67 List<String> rows = generateRandomWords(10, "row");
68 List<String> allColumns = generateRandomWords(10, "column");
69 List<String> values = generateRandomWords(100, "value");
70
71 long maxTimestamp = 2;
72 double selectPercent = 0.5;
73 int numberOfTests = 5;
74 double flushPercentage = 0.2;
75 double minorPercentage = 0.2;
76 double majorPercentage = 0.2;
77 double putPercentage = 0.2;
78
79 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
80
81 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
82 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
83
84 for (int i = 0; i < numberOfTests; i++) {
85 kvMaps[i] = new HashMap<String, KeyValue>();
86 columnLists[i] = new ArrayList<String>();
87 for (String column : allColumns) {
88 if (Math.random() < selectPercent) {
89 columnLists[i].add(column);
90 }
91 }
92 }
93
94 for (String value : values) {
95 for (String row : rows) {
96 Put p = new Put(Bytes.toBytes(row));
97 p.setDurability(Durability.SKIP_WAL);
98 for (String column : allColumns) {
99 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
100 KeyValue kv =
101 KeyValueTestUtil.create(row, family, column, timestamp, value);
102 if (Math.random() < putPercentage) {
103 p.add(kv);
104 allKVMap.put(kv.getKeyString(), kv);
105 for (int i = 0; i < numberOfTests; i++) {
106 if (columnLists[i].contains(column)) {
107 kvMaps[i].put(kv.getKeyString(), kv);
108 }
109 }
110 }
111 }
112 }
113 region.put(p);
114 if (Math.random() < flushPercentage) {
115 LOG.info("Flushing... ");
116 region.flushcache();
117 }
118
119 if (Math.random() < minorPercentage) {
120 LOG.info("Minor compacting... ");
121 region.compactStores(false);
122 }
123
124 if (Math.random() < majorPercentage) {
125 LOG.info("Major compacting... ");
126 region.compactStores(true);
127 }
128 }
129 }
130
131 for (int i = 0; i < numberOfTests + 1; i++) {
132 Collection<KeyValue> kvSet;
133 Scan scan = new Scan();
134 scan.setMaxVersions();
135 if (i < numberOfTests) {
136 if (columnLists[i].size() == 0) continue;
137 kvSet = kvMaps[i].values();
138 for (String column : columnLists[i]) {
139 scan.addColumn(familyBytes, Bytes.toBytes(column));
140 }
141 LOG.info("ExplicitColumns scanner");
142 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
143 + kvSet.size());
144 } else {
145 kvSet = allKVMap.values();
146 LOG.info("Wildcard scanner");
147 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
148
149 }
150 InternalScanner scanner = region.getScanner(scan);
151 List<KeyValue> results = new ArrayList<KeyValue>();
152 while (scanner.next(results))
153 ;
154 assertEquals(kvSet.size(), results.size());
155 assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
156 }
157 } finally {
158 HRegion.closeHRegion(region);
159 }
160
161 HRegion.closeHRegion(region);
162 }
163
164 @SuppressWarnings("unchecked")
165 @Test
166 public void testReseeking() throws IOException {
167 String family = "Family";
168 byte[] familyBytes = Bytes.toBytes("Family");
169 TableName table =
170 TableName.valueOf("TestSingleVersions");
171
172 HTableDescriptor htd = new HTableDescriptor(table);
173 HColumnDescriptor hcd = new HColumnDescriptor(family);
174 hcd.setMaxVersions(3);
175 htd.addFamily(hcd);
176
177 HRegionInfo info = new HRegionInfo(table, null, null, false);
178 HRegion region =
179 HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL
180 .getConfiguration(), htd);
181
182 List<String> rows = generateRandomWords(10, "row");
183 List<String> allColumns = generateRandomWords(100, "column");
184
185 long maxTimestamp = 2;
186 double selectPercent = 0.5;
187 int numberOfTests = 5;
188 double flushPercentage = 0.2;
189 double minorPercentage = 0.2;
190 double majorPercentage = 0.2;
191 double putPercentage = 0.2;
192
193 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
194
195 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
196 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
197 String valueString = "Value";
198
199 for (int i = 0; i < numberOfTests; i++) {
200 kvMaps[i] = new HashMap<String, KeyValue>();
201 columnLists[i] = new ArrayList<String>();
202 for (String column : allColumns) {
203 if (Math.random() < selectPercent) {
204 columnLists[i].add(column);
205 }
206 }
207 }
208
209 for (String row : rows) {
210 Put p = new Put(Bytes.toBytes(row));
211 p.setDurability(Durability.SKIP_WAL);
212 for (String column : allColumns) {
213 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
214 KeyValue kv =
215 KeyValueTestUtil.create(row, family, column, timestamp,
216 valueString);
217 if (Math.random() < putPercentage) {
218 p.add(kv);
219 allKVMap.put(kv.getKeyString(), kv);
220 for (int i = 0; i < numberOfTests; i++) {
221 if (columnLists[i].contains(column)) {
222 kvMaps[i].put(kv.getKeyString(), kv);
223 }
224 }
225 }
226
227 }
228 }
229 region.put(p);
230 if (Math.random() < flushPercentage) {
231 LOG.info("Flushing... ");
232 region.flushcache();
233 }
234
235 if (Math.random() < minorPercentage) {
236 LOG.info("Minor compacting... ");
237 region.compactStores(false);
238 }
239
240 if (Math.random() < majorPercentage) {
241 LOG.info("Major compacting... ");
242 region.compactStores(true);
243 }
244 }
245
246 for (int i = 0; i < numberOfTests + 1; i++) {
247 Collection<KeyValue> kvSet;
248 Scan scan = new Scan();
249 scan.setMaxVersions();
250 if (i < numberOfTests) {
251 if (columnLists[i].size() == 0) continue;
252 kvSet = kvMaps[i].values();
253 for (String column : columnLists[i]) {
254 scan.addColumn(familyBytes, Bytes.toBytes(column));
255 }
256 LOG.info("ExplicitColumns scanner");
257 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
258 + kvSet.size());
259 } else {
260 kvSet = allKVMap.values();
261 LOG.info("Wildcard scanner");
262 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
263
264 }
265 InternalScanner scanner = region.getScanner(scan);
266 List<KeyValue> results = new ArrayList<KeyValue>();
267 while (scanner.next(results))
268 ;
269 assertEquals(kvSet.size(), results.size());
270 assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
271 }
272
273 HRegion.closeHRegion(region);
274 }
275
276 List<String> generateRandomWords(int numberOfWords, String suffix) {
277 Set<String> wordSet = new HashSet<String>();
278 for (int i = 0; i < numberOfWords; i++) {
279 int lengthOfWords = (int) (Math.random() * 5) + 1;
280 char[] wordChar = new char[lengthOfWords];
281 for (int j = 0; j < wordChar.length; j++) {
282 wordChar[j] = (char) (Math.random() * 26 + 97);
283 }
284 String word;
285 if (suffix == null) {
286 word = new String(wordChar);
287 } else {
288 word = new String(wordChar) + suffix;
289 }
290 wordSet.add(word);
291 }
292 List<String> wordList = new ArrayList<String>(wordSet);
293 return wordList;
294 }
295
296 }
297