1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.util;
21
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25 import java.util.Arrays;
26 import java.util.LinkedList;
27 import java.util.Queue;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.io.hfile.BlockType;
32 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex;
33 import org.apache.hadoop.hbase.io.hfile.InlineBlockWriter;
34 import org.apache.hadoop.io.RawComparator;
35 import org.apache.hadoop.io.Writable;
36
37
38
39
40
41
42 public class CompoundBloomFilterWriter extends CompoundBloomFilterBase
43 implements BloomFilterWriter, InlineBlockWriter {
44
45 protected static final Log LOG =
46 LogFactory.getLog(CompoundBloomFilterWriter.class);
47
48
49 private ByteBloomFilter chunk;
50
51
52 private ByteBloomFilter prevChunk;
53
54
55 private int maxFold;
56
57
58 private int chunkByteSize;
59
60
61 private static class ReadyChunk {
62 int chunkId;
63 byte[] firstKey;
64 ByteBloomFilter chunk;
65 }
66
67 private Queue<ReadyChunk> readyChunks = new LinkedList<ReadyChunk>();
68
69
70 private byte[] firstKeyInChunk = null;
71
72 private HFileBlockIndex.BlockIndexWriter bloomBlockIndexWriter =
73 new HFileBlockIndex.BlockIndexWriter();
74
75
76 private boolean cacheOnWrite;
77
78
79
80
81
82
83
84
85
86
87
88
89 public CompoundBloomFilterWriter(int chunkByteSizeHint, float errorRate,
90 int hashType, int maxFold, boolean cacheOnWrite,
91 RawComparator<byte[]> comparator) {
92 chunkByteSize = ByteBloomFilter.computeFoldableByteSize(
93 chunkByteSizeHint * 8, maxFold);
94
95 this.errorRate = errorRate;
96 this.hashType = hashType;
97 this.maxFold = maxFold;
98 this.cacheOnWrite = cacheOnWrite;
99 this.comparator = comparator;
100 }
101
102 @Override
103 public boolean shouldWriteBlock(boolean closing) {
104 enqueueReadyChunk(closing);
105 return !readyChunks.isEmpty();
106 }
107
108
109
110
111
112
113
114 private void enqueueReadyChunk(boolean closing) {
115 if (chunk == null ||
116 (chunk.getKeyCount() < chunk.getMaxKeys() && !closing)) {
117 return;
118 }
119
120 if (firstKeyInChunk == null) {
121 throw new NullPointerException("Trying to enqueue a chunk, " +
122 "but first key is null: closing=" + closing + ", keyCount=" +
123 chunk.getKeyCount() + ", maxKeys=" + chunk.getMaxKeys());
124 }
125
126 ReadyChunk readyChunk = new ReadyChunk();
127 readyChunk.chunkId = numChunks - 1;
128 readyChunk.chunk = chunk;
129 readyChunk.firstKey = firstKeyInChunk;
130 readyChunks.add(readyChunk);
131
132 long prevMaxKeys = chunk.getMaxKeys();
133 long prevByteSize = chunk.getByteSize();
134
135 chunk.compactBloom();
136
137 if (LOG.isDebugEnabled() && prevByteSize != chunk.getByteSize()) {
138 LOG.debug("Compacted Bloom chunk #" + readyChunk.chunkId + " from ["
139 + prevMaxKeys + " max keys, " + prevByteSize + " bytes] to ["
140 + chunk.getMaxKeys() + " max keys, " + chunk.getByteSize()
141 + " bytes]");
142 }
143
144 totalMaxKeys += chunk.getMaxKeys();
145 totalByteSize += chunk.getByteSize();
146
147 firstKeyInChunk = null;
148 prevChunk = chunk;
149 chunk = null;
150 }
151
152
153
154
155
156
157
158
159 @Override
160 public void add(byte[] bloomKey, int keyOffset, int keyLength) {
161 if (bloomKey == null)
162 throw new NullPointerException();
163
164 enqueueReadyChunk(false);
165
166 if (chunk == null) {
167 if (firstKeyInChunk != null) {
168 throw new IllegalStateException("First key in chunk already set: "
169 + Bytes.toStringBinary(firstKeyInChunk));
170 }
171 firstKeyInChunk = Arrays.copyOfRange(bloomKey, keyOffset, keyOffset
172 + keyLength);
173
174 if (prevChunk == null) {
175
176 chunk = ByteBloomFilter.createBySize(chunkByteSize, errorRate,
177 hashType, maxFold);
178 } else {
179
180
181 chunk = prevChunk.createAnother();
182 }
183
184 if (chunk.getKeyCount() != 0) {
185 throw new IllegalStateException("keyCount=" + chunk.getKeyCount()
186 + " > 0");
187 }
188
189 chunk.allocBloom();
190 ++numChunks;
191 }
192
193 chunk.add(bloomKey, keyOffset, keyLength);
194 ++totalKeyCount;
195 }
196
197 @Override
198 public void writeInlineBlock(DataOutput out) throws IOException {
199
200
201 ReadyChunk readyChunk = readyChunks.peek();
202
203 ByteBloomFilter readyChunkBloom = readyChunk.chunk;
204 readyChunkBloom.getDataWriter().write(out);
205 }
206
207 @Override
208 public void blockWritten(long offset, int onDiskSize, int uncompressedSize) {
209 ReadyChunk readyChunk = readyChunks.remove();
210 bloomBlockIndexWriter.addEntry(readyChunk.firstKey, offset, onDiskSize);
211 }
212
213 @Override
214 public BlockType getInlineBlockType() {
215 return BlockType.BLOOM_CHUNK;
216 }
217
218 private class MetaWriter implements Writable {
219 protected MetaWriter() {}
220
221 @Override
222 public void readFields(DataInput in) throws IOException {
223 throw new IOException("Cant read with this class.");
224 }
225
226
227
228
229
230
231
232
233 @Override
234 public void write(DataOutput out) throws IOException {
235 out.writeInt(VERSION);
236
237 out.writeLong(getByteSize());
238 out.writeInt(prevChunk.getHashCount());
239 out.writeInt(prevChunk.getHashType());
240 out.writeLong(getKeyCount());
241 out.writeLong(getMaxKeys());
242
243
244 out.writeInt(numChunks);
245 Bytes.writeByteArray(out,
246 Bytes.toBytes(comparator.getClass().getName()));
247
248
249 bloomBlockIndexWriter.writeSingleLevelIndex(out, "Bloom filter");
250 }
251 }
252
253 @Override
254 public Writable getMetaWriter() {
255 return new MetaWriter();
256 }
257
258 @Override
259 public void compactBloom() {
260 }
261
262 @Override
263 public void allocBloom() {
264
265 }
266
267 @Override
268 public Writable getDataWriter() {
269 return null;
270 }
271
272 @Override
273 public boolean cacheOnWrite() {
274 return cacheOnWrite;
275 }
276
277 }