1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collection;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.List;
28 import java.util.NavigableSet;
29 import java.util.TreeSet;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.KeyValueTestUtil;
40 import org.apache.hadoop.hbase.SmallTests;
41 import org.apache.hadoop.hbase.client.Put;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.io.compress.Compression;
44 import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
45 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.junit.Before;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50 import org.junit.runner.RunWith;
51 import org.junit.runners.Parameterized;
52 import org.junit.runners.Parameterized.Parameters;
53
54 import static org.junit.Assert.*;
55
56
57
58
59
60 @RunWith(Parameterized.class)
61 @Category(SmallTests.class)
62 public class TestScanWithBloomError {
63
64 private static final Log LOG =
65 LogFactory.getLog(TestScanWithBloomError.class);
66
67 private static final String TABLE_NAME = "ScanWithBloomError";
68 private static final String FAMILY = "myCF";
69 private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
70 private static final String ROW = "theRow";
71 private static final String QUALIFIER_PREFIX = "qual";
72 private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
73 private static NavigableSet<Integer> allColIds = new TreeSet<Integer>();
74 private HRegion region;
75 private BloomType bloomType;
76 private FileSystem fs;
77 private Configuration conf;
78
79 private final static HBaseTestingUtility TEST_UTIL =
80 new HBaseTestingUtility();
81
82 @Parameters
83 public static final Collection<Object[]> parameters() {
84 List<Object[]> configurations = new ArrayList<Object[]>();
85 for (BloomType bloomType : BloomType.values()) {
86 configurations.add(new Object[] { bloomType });
87 }
88 return configurations;
89 }
90
91 public TestScanWithBloomError(BloomType bloomType) {
92 this.bloomType = bloomType;
93 }
94
95 @Before
96 public void setUp() throws IOException{
97 conf = TEST_UTIL.getConfiguration();
98 fs = FileSystem.get(conf);
99 }
100
101 @Test
102 public void testThreeStoreFiles() throws IOException {
103 region = TEST_UTIL.createTestRegion(TABLE_NAME,
104 new HColumnDescriptor(FAMILY)
105 .setCompressionType(Compression.Algorithm.GZ)
106 .setBloomFilterType(bloomType)
107 .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
108 createStoreFile(new int[] {1, 2, 6});
109 createStoreFile(new int[] {1, 2, 3, 7});
110 createStoreFile(new int[] {1, 9});
111 scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7});
112
113 HRegion.closeHRegion(region);
114 }
115
116 private void scanColSet(int[] colSet, int[] expectedResultCols)
117 throws IOException {
118 LOG.info("Scanning column set: " + Arrays.toString(colSet));
119 Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
120 addColumnSetToScan(scan, colSet);
121 RegionScannerImpl scanner = (RegionScannerImpl) region.getScanner(scan);
122 KeyValueHeap storeHeap = scanner.getStoreHeapForTesting();
123 assertEquals(0, storeHeap.getHeap().size());
124 StoreScanner storeScanner =
125 (StoreScanner) storeHeap.getCurrentForTesting();
126 @SuppressWarnings({ "unchecked", "rawtypes" })
127 List<StoreFileScanner> scanners = (List<StoreFileScanner>)
128 (List) storeScanner.getAllScannersForTesting();
129
130
131 Collections.sort(scanners, new Comparator<StoreFileScanner>() {
132 @Override
133 public int compare(StoreFileScanner s1, StoreFileScanner s2) {
134 Path p1 = s1.getReaderForTesting().getHFileReader().getPath();
135 Path p2 = s2.getReaderForTesting().getHFileReader().getPath();
136 long t1, t2;
137 try {
138 t1 = fs.getFileStatus(p1).getModificationTime();
139 t2 = fs.getFileStatus(p2).getModificationTime();
140 } catch (IOException ex) {
141 throw new RuntimeException(ex);
142 }
143 return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
144 }
145 });
146
147 StoreFile.Reader lastStoreFileReader = null;
148 for (StoreFileScanner sfScanner : scanners)
149 lastStoreFileReader = sfScanner.getReaderForTesting();
150
151 new HFilePrettyPrinter().run(new String[]{ "-m", "-p", "-f",
152 lastStoreFileReader.getHFileReader().getPath().toString()});
153
154
155
156 LOG.info("Disabling Bloom filter for: "
157 + lastStoreFileReader.getHFileReader().getName());
158 lastStoreFileReader.disableBloomFilterForTesting();
159
160 List<KeyValue> allResults = new ArrayList<KeyValue>();
161
162 {
163 List<KeyValue> results = new ArrayList<KeyValue>();
164 while (scanner.next(results) || results.size() > 0) {
165 allResults.addAll(results);
166 results.clear();
167 }
168 }
169
170 List<Integer> actualIds = new ArrayList<Integer>();
171 for (KeyValue kv : allResults) {
172 String qual = Bytes.toString(kv.getQualifier());
173 assertTrue(qual.startsWith(QUALIFIER_PREFIX));
174 actualIds.add(Integer.valueOf(qual.substring(
175 QUALIFIER_PREFIX.length())));
176 }
177 List<Integer> expectedIds = new ArrayList<Integer>();
178 for (int expectedId : expectedResultCols)
179 expectedIds.add(expectedId);
180
181 LOG.info("Column ids returned: " + actualIds + ", expected: "
182 + expectedIds);
183 assertEquals(expectedIds.toString(), actualIds.toString());
184 }
185
186 private void addColumnSetToScan(Scan scan, int[] colIds) {
187 for (int colId : colIds) {
188 scan.addColumn(FAMILY_BYTES,
189 Bytes.toBytes(qualFromId(colId)));
190 }
191 }
192
193 private String qualFromId(int colId) {
194 return QUALIFIER_PREFIX + colId;
195 }
196
197 private void createStoreFile(int[] colIds)
198 throws IOException {
199 Put p = new Put(ROW_BYTES);
200 for (int colId : colIds) {
201 long ts = Long.MAX_VALUE;
202 String qual = qualFromId(colId);
203 allColIds.add(colId);
204 KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY,
205 qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts));
206 p.add(kv);
207 }
208 region.put(p);
209 region.flushcache();
210 }
211
212
213 }
214