1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HBaseConfiguration;
32 import org.apache.hadoop.hbase.HBaseTestCase;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.KeyValue;
38 import org.apache.hadoop.hbase.MediumTests;
39 import org.apache.hadoop.hbase.client.Delete;
40 import org.apache.hadoop.hbase.client.Get;
41 import org.apache.hadoop.hbase.client.Put;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.io.hfile.BlockCache;
44 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
45 import org.apache.hadoop.hbase.io.hfile.HFile;
46 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52 @Category(MediumTests.class)
53 public class TestBlocksRead extends HBaseTestCase {
54 static final Log LOG = LogFactory.getLog(TestBlocksRead.class);
55 static final BloomType[] BLOOM_TYPE = new BloomType[] { BloomType.ROWCOL,
56 BloomType.ROW, BloomType.NONE };
57
58 private static BlockCache blockCache;
59
60 private HBaseConfiguration getConf() {
61 HBaseConfiguration conf = new HBaseConfiguration();
62
63
64 conf.setInt("hbase.hstore.compactionThreshold", 10000);
65 return conf;
66 }
67
68 HRegion region = null;
69 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
70 private final String DIR = TEST_UTIL.getDataTestDir("TestBlocksRead").toString();
71
72
73
74
75 @SuppressWarnings("deprecation")
76 @Override
77 protected void setUp() throws Exception {
78 super.setUp();
79 }
80
81 @SuppressWarnings("deprecation")
82 @Override
83 protected void tearDown() throws Exception {
84 super.tearDown();
85 EnvironmentEdgeManagerTestHelper.reset();
86 }
87
88
89
90
91
92
93
94
95
96
97 private HRegion initHRegion(byte[] tableName, String callingMethod,
98 HBaseConfiguration conf, String family) throws IOException {
99 HTableDescriptor htd = new HTableDescriptor(tableName);
100 HColumnDescriptor familyDesc;
101 for (int i = 0; i < BLOOM_TYPE.length; i++) {
102 BloomType bloomType = BLOOM_TYPE[i];
103 familyDesc = new HColumnDescriptor(family + "_" + bloomType)
104 .setBlocksize(1)
105 .setBloomFilterType(BLOOM_TYPE[i]);
106 htd.addFamily(familyDesc);
107 }
108
109 HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
110 Path path = new Path(DIR + callingMethod);
111 HRegion r = HRegion.createHRegion(info, path, conf, htd);
112 blockCache = new CacheConfig(conf).getBlockCache();
113 return r;
114 }
115
116 private void putData(String family, String row, String col, long version)
117 throws IOException {
118 for (int i = 0; i < BLOOM_TYPE.length; i++) {
119 putData(Bytes.toBytes(family + "_" + BLOOM_TYPE[i]), row, col, version,
120 version);
121 }
122 }
123
124
125 private static byte[] genValue(String row, String col, long version) {
126 return Bytes.toBytes("Value:" + row + "#" + col + "#" + version);
127 }
128
129 private void putData(byte[] cf, String row, String col, long versionStart,
130 long versionEnd) throws IOException {
131 byte columnBytes[] = Bytes.toBytes(col);
132 Put put = new Put(Bytes.toBytes(row));
133 put.setWriteToWAL(false);
134
135 for (long version = versionStart; version <= versionEnd; version++) {
136 put.add(cf, columnBytes, version, genValue(row, col, version));
137 }
138 region.put(put);
139 }
140
141 private KeyValue[] getData(String family, String row, List<String> columns,
142 int expBlocks) throws IOException {
143 return getData(family, row, columns, expBlocks, expBlocks, expBlocks);
144 }
145
146 private KeyValue[] getData(String family, String row, List<String> columns,
147 int expBlocksRowCol, int expBlocksRow, int expBlocksNone)
148 throws IOException {
149 int[] expBlocks = new int[] { expBlocksRowCol, expBlocksRow, expBlocksNone };
150 KeyValue[] kvs = null;
151
152 for (int i = 0; i < BLOOM_TYPE.length; i++) {
153 BloomType bloomType = BLOOM_TYPE[i];
154 byte[] cf = Bytes.toBytes(family + "_" + bloomType);
155 long blocksStart = getBlkAccessCount(cf);
156 Get get = new Get(Bytes.toBytes(row));
157
158 for (String column : columns) {
159 get.addColumn(cf, Bytes.toBytes(column));
160 }
161
162 kvs = region.get(get, null).raw();
163 long blocksEnd = getBlkAccessCount(cf);
164 if (expBlocks[i] != -1) {
165 assertEquals("Blocks Read Check for Bloom: " + bloomType, expBlocks[i],
166 blocksEnd - blocksStart);
167 }
168 System.out.println("Blocks Read for Bloom: " + bloomType + " = "
169 + (blocksEnd - blocksStart) + "Expected = " + expBlocks[i]);
170 }
171 return kvs;
172 }
173
174 private KeyValue[] getData(String family, String row, String column,
175 int expBlocks) throws IOException {
176 return getData(family, row, Arrays.asList(column), expBlocks, expBlocks,
177 expBlocks);
178 }
179
180 private KeyValue[] getData(String family, String row, String column,
181 int expBlocksRowCol, int expBlocksRow, int expBlocksNone)
182 throws IOException {
183 return getData(family, row, Arrays.asList(column), expBlocksRowCol,
184 expBlocksRow, expBlocksNone);
185 }
186
187 private void deleteFamily(String family, String row, long version)
188 throws IOException {
189 Delete del = new Delete(Bytes.toBytes(row));
190 del.deleteFamily(Bytes.toBytes(family + "_ROWCOL"), version);
191 del.deleteFamily(Bytes.toBytes(family + "_ROW"), version);
192 del.deleteFamily(Bytes.toBytes(family + "_NONE"), version);
193 region.delete(del, null, true);
194 }
195
196 private static void verifyData(KeyValue kv, String expectedRow,
197 String expectedCol, long expectedVersion) {
198 assertEquals("RowCheck", expectedRow, Bytes.toString(kv.getRow()));
199 assertEquals("ColumnCheck", expectedCol, Bytes.toString(kv.getQualifier()));
200 assertEquals("TSCheck", expectedVersion, kv.getTimestamp());
201 assertEquals("ValueCheck",
202 Bytes.toString(genValue(expectedRow, expectedCol, expectedVersion)),
203 Bytes.toString(kv.getValue()));
204 }
205
206 private static long getBlkAccessCount(byte[] cf) {
207 return HFile.dataBlockReadCnt.get();
208 }
209
210 private static long getBlkCount() {
211 return blockCache.getBlockCount();
212 }
213
214
215
216
217
218
219 @Test
220 public void testBlocksRead() throws Exception {
221 byte[] TABLE = Bytes.toBytes("testBlocksRead");
222 String FAMILY = "cf1";
223 KeyValue kvs[];
224 HBaseConfiguration conf = getConf();
225 this.region = initHRegion(TABLE, getName(), conf, FAMILY);
226
227 try {
228 putData(FAMILY, "row", "col1", 1);
229 putData(FAMILY, "row", "col2", 2);
230 putData(FAMILY, "row", "col3", 3);
231 putData(FAMILY, "row", "col4", 4);
232 putData(FAMILY, "row", "col5", 5);
233 putData(FAMILY, "row", "col6", 6);
234 putData(FAMILY, "row", "col7", 7);
235 region.flushcache();
236
237
238
239
240 kvs = getData(FAMILY, "row", "col1", 1);
241 assertEquals(1, kvs.length);
242 verifyData(kvs[0], "row", "col1", 1);
243
244
245
246
247 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
248 assertEquals(2, kvs.length);
249 verifyData(kvs[0], "row", "col1", 1);
250 verifyData(kvs[1], "row", "col2", 2);
251
252
253
254
255
256 kvs = getData(FAMILY, "row", Arrays.asList("col2", "col3"), 3);
257 assertEquals(2, kvs.length);
258 verifyData(kvs[0], "row", "col2", 2);
259 verifyData(kvs[1], "row", "col3", 3);
260
261
262 kvs = getData(FAMILY, "row", Arrays.asList("col5"), 2);
263 assertEquals(1, kvs.length);
264 verifyData(kvs[0], "row", "col5", 5);
265 } finally {
266 HRegion.closeHRegion(this.region);
267 this.region = null;
268 }
269 }
270
271
272
273
274
275
276 @Test
277 public void testLazySeekBlocksRead() throws Exception {
278 byte[] TABLE = Bytes.toBytes("testLazySeekBlocksRead");
279 String FAMILY = "cf1";
280 KeyValue kvs[];
281 HBaseConfiguration conf = getConf();
282 this.region = initHRegion(TABLE, getName(), conf, FAMILY);
283
284 try {
285
286 putData(FAMILY, "row", "col1", 1);
287 putData(FAMILY, "row", "col2", 2);
288 region.flushcache();
289
290
291 putData(FAMILY, "row", "col1", 3);
292 putData(FAMILY, "row", "col2", 4);
293 region.flushcache();
294
295
296
297
298 kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1);
299 assertEquals(1, kvs.length);
300 verifyData(kvs[0], "row", "col1", 3);
301
302
303
304
305
306 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
307 assertEquals(2, kvs.length);
308 verifyData(kvs[0], "row", "col1", 3);
309 verifyData(kvs[1], "row", "col2", 4);
310
311
312 putData(FAMILY, "row", "col3", 5);
313 region.flushcache();
314
315
316
317
318 kvs = getData(FAMILY, "row", "col3", 1);
319 assertEquals(1, kvs.length);
320 verifyData(kvs[0], "row", "col3", 5);
321
322
323
324
325
326 kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1, 2, 2);
327 assertEquals(1, kvs.length);
328 verifyData(kvs[0], "row", "col1", 3);
329
330
331 deleteFamily(FAMILY, "row", 6);
332 region.flushcache();
333
334
335
336
337 kvs = getData(FAMILY, "row", "col1", 2, 3, 3);
338 assertEquals(0, kvs.length);
339 kvs = getData(FAMILY, "row", "col2", 3, 4, 4);
340 assertEquals(0, kvs.length);
341 kvs = getData(FAMILY, "row", "col3", 2);
342 assertEquals(0, kvs.length);
343 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 4);
344 assertEquals(0, kvs.length);
345
346
347 deleteFamily(FAMILY, "row", 10);
348 region.flushcache();
349
350
351
352 putData(FAMILY, "row", "col1", 7);
353 putData(FAMILY, "row", "col2", 8);
354 putData(FAMILY, "row", "col3", 9);
355 region.flushcache();
356
357
358 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5);
359 assertEquals(0, kvs.length);
360
361
362 putData(FAMILY, "row", "col1", 11);
363 putData(FAMILY, "row", "col2", 12);
364 putData(FAMILY, "row", "col3", 13);
365 region.flushcache();
366
367
368
369 kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5);
370 assertEquals(3, kvs.length);
371 verifyData(kvs[0], "row", "col1", 11);
372 verifyData(kvs[1], "row", "col2", 12);
373 verifyData(kvs[2], "row", "col3", 13);
374 } finally {
375 HRegion.closeHRegion(this.region);
376 this.region = null;
377 }
378 }
379
380
381
382
383
384 @Test
385 public void testBlocksStoredWhenCachingDisabled() throws Exception {
386 byte [] TABLE = Bytes.toBytes("testBlocksReadWhenCachingDisabled");
387 String FAMILY = "cf1";
388
389 HBaseConfiguration conf = getConf();
390 this.region = initHRegion(TABLE, getName(), conf, FAMILY);
391
392 try {
393 putData(FAMILY, "row", "col1", 1);
394 putData(FAMILY, "row", "col2", 2);
395 region.flushcache();
396
397
398
399 long blocksStart = getBlkCount();
400 Scan scan = new Scan();
401 scan.setCacheBlocks(false);
402 RegionScanner rs = region.getScanner(scan);
403 List<KeyValue> result = new ArrayList<KeyValue>(2);
404 rs.next(result);
405 assertEquals(2 * BLOOM_TYPE.length, result.size());
406 rs.close();
407 long blocksEnd = getBlkCount();
408
409 assertEquals(blocksStart, blocksEnd);
410
411
412
413 blocksStart = blocksEnd;
414 scan.setCacheBlocks(true);
415 rs = region.getScanner(scan);
416 result = new ArrayList<KeyValue>(2);
417 rs.next(result);
418 assertEquals(2 * BLOOM_TYPE.length, result.size());
419 rs.close();
420 blocksEnd = getBlkCount();
421
422 assertEquals(2 * BLOOM_TYPE.length, blocksEnd - blocksStart);
423 } finally {
424 HRegion.closeHRegion(this.region);
425 this.region = null;
426 }
427 }
428
429 @Test
430 public void testLazySeekBlocksReadWithDelete() throws Exception {
431 byte[] TABLE = Bytes.toBytes("testLazySeekBlocksReadWithDelete");
432 String FAMILY = "cf1";
433 KeyValue kvs[];
434 HBaseConfiguration conf = getConf();
435 this.region = initHRegion(TABLE, getName(), conf, FAMILY);
436 try {
437 deleteFamily(FAMILY, "row", 200);
438 for (int i = 0; i < 100; i++) {
439 putData(FAMILY, "row", "col" + i, i);
440 }
441 putData(FAMILY, "row", "col99", 201);
442 region.flushcache();
443
444 kvs = getData(FAMILY, "row", Arrays.asList("col0"), 2);
445 assertEquals(0, kvs.length);
446
447 kvs = getData(FAMILY, "row", Arrays.asList("col99"), 2);
448 assertEquals(1, kvs.length);
449 verifyData(kvs[0], "row", "col99", 201);
450 } finally {
451 HRegion.closeHRegion(this.region);
452 this.region = null;
453 }
454 }
455
456 @org.junit.Rule
457 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
458 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
459 }