1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.Collection;
26 import java.util.Collections;
27 import java.util.Comparator;
28 import java.util.List;
29 import java.util.NavigableSet;
30 import java.util.TreeSet;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.KeyValue;
40 import org.apache.hadoop.hbase.KeyValueTestUtil;
41 import org.apache.hadoop.hbase.SmallTests;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Scan;
44 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
45 import org.apache.hadoop.hbase.io.hfile.Compression;
46 import org.apache.hadoop.hbase.io.hfile.HFile;
47 import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
48 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.junit.Before;
51 import org.junit.Test;
52 import org.junit.experimental.categories.Category;
53 import org.junit.runner.RunWith;
54 import org.junit.runners.Parameterized;
55 import org.junit.runners.Parameterized.Parameters;
56
57 import static org.junit.Assert.*;
58
59
60
61
62
63 @RunWith(Parameterized.class)
64 @Category(SmallTests.class)
65 public class TestScanWithBloomError {
66
67 private static final Log LOG =
68 LogFactory.getLog(TestScanWithBloomError.class);
69
70 private static final String TABLE_NAME = "ScanWithBloomError";
71 private static final String FAMILY = "myCF";
72 private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
73 private static final String ROW = "theRow";
74 private static final String QUALIFIER_PREFIX = "qual";
75 private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
76 private static NavigableSet<Integer> allColIds = new TreeSet<Integer>();
77 private HRegion region;
78 private StoreFile.BloomType bloomType;
79 private FileSystem fs;
80 private Configuration conf;
81
82 private final static HBaseTestingUtility TEST_UTIL =
83 new HBaseTestingUtility();
84
85 @Parameters
86 public static final Collection<Object[]> parameters() {
87 List<Object[]> configurations = new ArrayList<Object[]>();
88 for (StoreFile.BloomType bloomType : StoreFile.BloomType.values()) {
89 configurations.add(new Object[] { bloomType });
90 }
91 return configurations;
92 }
93
94 public TestScanWithBloomError(StoreFile.BloomType bloomType) {
95 this.bloomType = bloomType;
96 }
97
98 @Before
99 public void setUp() throws IOException{
100 conf = TEST_UTIL.getConfiguration();
101 fs = FileSystem.get(conf);
102 }
103
104 @Test
105 public void testThreeStoreFiles() throws IOException {
106 region = TEST_UTIL.createTestRegion(TABLE_NAME,
107 new HColumnDescriptor(FAMILY)
108 .setCompressionType(Compression.Algorithm.GZ)
109 .setBloomFilterType(bloomType)
110 .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
111 createStoreFile(new int[] {1, 2, 6});
112 createStoreFile(new int[] {1, 2, 3, 7});
113 createStoreFile(new int[] {1, 9});
114 scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7});
115
116 region.close();
117 region.getLog().closeAndDelete();
118 }
119
120 private void scanColSet(int[] colSet, int[] expectedResultCols)
121 throws IOException {
122 LOG.info("Scanning column set: " + Arrays.toString(colSet));
123 Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
124 addColumnSetToScan(scan, colSet);
125 RegionScannerImpl scanner = (RegionScannerImpl) region.getScanner(scan);
126 KeyValueHeap storeHeap = scanner.getStoreHeapForTesting();
127 assertEquals(0, storeHeap.getHeap().size());
128 StoreScanner storeScanner =
129 (StoreScanner) storeHeap.getCurrentForTesting();
130 @SuppressWarnings({ "unchecked", "rawtypes" })
131 List<StoreFileScanner> scanners = (List<StoreFileScanner>)
132 (List) storeScanner.getAllScannersForTesting();
133
134
135 Collections.sort(scanners, new Comparator<StoreFileScanner>() {
136 @Override
137 public int compare(StoreFileScanner s1, StoreFileScanner s2) {
138 Path p1 = s1.getReaderForTesting().getHFileReader().getPath();
139 Path p2 = s2.getReaderForTesting().getHFileReader().getPath();
140 long t1, t2;
141 try {
142 t1 = fs.getFileStatus(p1).getModificationTime();
143 t2 = fs.getFileStatus(p2).getModificationTime();
144 } catch (IOException ex) {
145 throw new RuntimeException(ex);
146 }
147 return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
148 }
149 });
150
151 StoreFile.Reader lastStoreFileReader = null;
152 for (StoreFileScanner sfScanner : scanners)
153 lastStoreFileReader = sfScanner.getReaderForTesting();
154
155 new HFilePrettyPrinter().run(new String[]{ "-m", "-p", "-f",
156 lastStoreFileReader.getHFileReader().getPath().toString()});
157
158
159
160 LOG.info("Disabling Bloom filter for: "
161 + lastStoreFileReader.getHFileReader().getName());
162 lastStoreFileReader.disableBloomFilterForTesting();
163
164 List<KeyValue> allResults = new ArrayList<KeyValue>();
165
166 {
167 List<KeyValue> results = new ArrayList<KeyValue>();
168 while (scanner.next(results) || results.size() > 0) {
169 allResults.addAll(results);
170 results.clear();
171 }
172 }
173
174 List<Integer> actualIds = new ArrayList<Integer>();
175 for (KeyValue kv : allResults) {
176 String qual = Bytes.toString(kv.getQualifier());
177 assertTrue(qual.startsWith(QUALIFIER_PREFIX));
178 actualIds.add(Integer.valueOf(qual.substring(
179 QUALIFIER_PREFIX.length())));
180 }
181 List<Integer> expectedIds = new ArrayList<Integer>();
182 for (int expectedId : expectedResultCols)
183 expectedIds.add(expectedId);
184
185 LOG.info("Column ids returned: " + actualIds + ", expected: "
186 + expectedIds);
187 assertEquals(expectedIds.toString(), actualIds.toString());
188 }
189
190 private void addColumnSetToScan(Scan scan, int[] colIds) {
191 for (int colId : colIds) {
192 scan.addColumn(FAMILY_BYTES,
193 Bytes.toBytes(qualFromId(colId)));
194 }
195 }
196
197 private String qualFromId(int colId) {
198 return QUALIFIER_PREFIX + colId;
199 }
200
201 private void createStoreFile(int[] colIds)
202 throws IOException {
203 Put p = new Put(ROW_BYTES);
204 for (int colId : colIds) {
205 long ts = Long.MAX_VALUE;
206 String qual = qualFromId(colId);
207 allColIds.add(colId);
208 KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY,
209 qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts));
210 p.add(kv);
211 }
212 region.put(p);
213 region.flushcache();
214 }
215
216
217 @org.junit.Rule
218 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
219 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
220 }
221