1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.encoding;
18
19 import java.io.DataInputStream;
20 import java.io.DataOutputStream;
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23
24 import org.apache.hadoop.classification.InterfaceAudience;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.KeyValue;
27 import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
28 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
29 import org.apache.hadoop.hbase.io.hfile.BlockType;
30 import org.apache.hadoop.hbase.util.ByteBufferUtils;
31 import org.apache.hadoop.hbase.util.Bytes;
32 import org.apache.hadoop.io.RawComparator;
33 import org.apache.hadoop.io.WritableUtils;
34
35
36
37
38 @InterfaceAudience.Private
39 abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
40
41 private static int INITIAL_KEY_BUFFER_SIZE = 512;
42
43 @Override
44 public ByteBuffer decodeKeyValues(DataInputStream source,
45 boolean includesMemstoreTS) throws IOException {
46 return decodeKeyValues(source, 0, 0, includesMemstoreTS);
47 }
48
49 protected static class SeekerState {
50 protected int valueOffset = -1;
51 protected int keyLength;
52 protected int valueLength;
53 protected int lastCommonPrefix;
54
55
56 protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
57
58 protected long memstoreTS;
59 protected int nextKvOffset;
60
61 protected boolean isValid() {
62 return valueOffset != -1;
63 }
64
65 protected void invalidate() {
66 valueOffset = -1;
67 }
68
69 protected void ensureSpaceForKey() {
70 if (keyLength > keyBuffer.length) {
71
72 int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
73 while (keyLength > newKeyBufferLength) {
74 newKeyBufferLength *= 2;
75 }
76 byte[] newKeyBuffer = new byte[newKeyBufferLength];
77 System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
78 keyBuffer = newKeyBuffer;
79 }
80 }
81
82
83
84
85
86
87 protected void copyFromNext(SeekerState nextState) {
88 if (keyBuffer.length != nextState.keyBuffer.length) {
89 keyBuffer = nextState.keyBuffer.clone();
90 } else if (!isValid()) {
91
92
93 System.arraycopy(nextState.keyBuffer, 0, keyBuffer, 0,
94 nextState.keyLength);
95 } else {
96
97 System.arraycopy(nextState.keyBuffer, nextState.lastCommonPrefix,
98 keyBuffer, nextState.lastCommonPrefix, nextState.keyLength
99 - nextState.lastCommonPrefix);
100 }
101
102 valueOffset = nextState.valueOffset;
103 keyLength = nextState.keyLength;
104 valueLength = nextState.valueLength;
105 lastCommonPrefix = nextState.lastCommonPrefix;
106 nextKvOffset = nextState.nextKvOffset;
107 memstoreTS = nextState.memstoreTS;
108 }
109
110 }
111
112 protected abstract static class
113 BufferedEncodedSeeker<STATE extends SeekerState>
114 implements EncodedSeeker {
115
116 protected final RawComparator<byte[]> comparator;
117 protected final SamePrefixComparator<byte[]> samePrefixComparator;
118 protected ByteBuffer currentBuffer;
119 protected STATE current = createSeekerState();
120 protected STATE previous = createSeekerState();
121
122 @SuppressWarnings("unchecked")
123 public BufferedEncodedSeeker(RawComparator<byte[]> comparator) {
124 this.comparator = comparator;
125 if (comparator instanceof SamePrefixComparator) {
126 this.samePrefixComparator = (SamePrefixComparator<byte[]>) comparator;
127 } else {
128 this.samePrefixComparator = null;
129 }
130 }
131
132 @Override
133 public void setCurrentBuffer(ByteBuffer buffer) {
134 currentBuffer = buffer;
135 decodeFirst();
136 previous.invalidate();
137 }
138
139 @Override
140 public ByteBuffer getKeyDeepCopy() {
141 ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
142 keyBuffer.put(current.keyBuffer, 0, current.keyLength);
143 return keyBuffer;
144 }
145
146 @Override
147 public ByteBuffer getValueShallowCopy() {
148 return ByteBuffer.wrap(currentBuffer.array(),
149 currentBuffer.arrayOffset() + current.valueOffset,
150 current.valueLength);
151 }
152
153 @Override
154 public ByteBuffer getKeyValueBuffer() {
155 ByteBuffer kvBuffer = ByteBuffer.allocate(
156 2 * Bytes.SIZEOF_INT + current.keyLength + current.valueLength);
157 kvBuffer.putInt(current.keyLength);
158 kvBuffer.putInt(current.valueLength);
159 kvBuffer.put(current.keyBuffer, 0, current.keyLength);
160 kvBuffer.put(currentBuffer.array(),
161 currentBuffer.arrayOffset() + current.valueOffset,
162 current.valueLength);
163 return kvBuffer;
164 }
165
166 @Override
167 public KeyValue getKeyValue() {
168 ByteBuffer kvBuf = getKeyValueBuffer();
169 KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset());
170 kv.setMemstoreTS(current.memstoreTS);
171 return kv;
172 }
173
174 @Override
175 public void rewind() {
176 currentBuffer.rewind();
177 decodeFirst();
178 previous.invalidate();
179 }
180
181 @Override
182 public boolean next() {
183 if (!currentBuffer.hasRemaining()) {
184 return false;
185 }
186 decodeNext();
187 previous.invalidate();
188 return true;
189 }
190
191 @Override
192 public int seekToKeyInBlock(byte[] key, int offset, int length,
193 boolean seekBefore) {
194 int commonPrefix = 0;
195 previous.invalidate();
196 do {
197 int comp;
198 if (samePrefixComparator != null) {
199 commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix);
200
201
202 commonPrefix += ByteBufferUtils.findCommonPrefix(
203 key, offset + commonPrefix, length - commonPrefix,
204 current.keyBuffer, commonPrefix,
205 current.keyLength - commonPrefix);
206
207 comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key,
208 offset, length, current.keyBuffer, 0, current.keyLength);
209 } else {
210 comp = comparator.compare(key, offset, length,
211 current.keyBuffer, 0, current.keyLength);
212 }
213
214 if (comp == 0) {
215 if (seekBefore) {
216 if (!previous.isValid()) {
217
218
219 throw new IllegalStateException("Cannot seekBefore if " +
220 "positioned at the first key in the block: key=" +
221 Bytes.toStringBinary(key, offset, length));
222 }
223 moveToPrevious();
224 return 1;
225 }
226 return 0;
227 }
228
229 if (comp < 0) {
230 if (previous.isValid()) {
231 moveToPrevious();
232 } else {
233 return HConstants.INDEX_KEY_MAGIC;
234 }
235 return 1;
236 }
237
238
239 if (currentBuffer.hasRemaining()) {
240 previous.copyFromNext(current);
241 decodeNext();
242 } else {
243 break;
244 }
245 } while (true);
246
247
248 return 1;
249 }
250
251 private void moveToPrevious() {
252 if (!previous.isValid()) {
253 throw new IllegalStateException(
254 "Can move back only once and not in first key in the block.");
255 }
256
257 STATE tmp = previous;
258 previous = current;
259 current = tmp;
260
261
262 currentBuffer.position(current.nextKvOffset);
263
264 previous.invalidate();
265 }
266
267 @SuppressWarnings("unchecked")
268 protected STATE createSeekerState() {
269
270
271 return (STATE) new SeekerState();
272 }
273
274 abstract protected void decodeFirst();
275 abstract protected void decodeNext();
276 }
277
278 protected final void afterEncodingKeyValue(ByteBuffer in,
279 DataOutputStream out, boolean includesMemstoreTS) {
280 if (includesMemstoreTS) {
281
282 long memstoreTS = -1;
283 try {
284 memstoreTS = ByteBufferUtils.readVLong(in);
285 WritableUtils.writeVLong(out, memstoreTS);
286 } catch (IOException ex) {
287 throw new RuntimeException("Unable to copy memstore timestamp " +
288 memstoreTS + " after encoding a key/value");
289 }
290 }
291 }
292
293 protected final void afterDecodingKeyValue(DataInputStream source,
294 ByteBuffer dest, boolean includesMemstoreTS) {
295 if (includesMemstoreTS) {
296 long memstoreTS = -1;
297 try {
298
299
300 memstoreTS = WritableUtils.readVLong(source);
301 ByteBufferUtils.writeVLong(dest, memstoreTS);
302 } catch (IOException ex) {
303 throw new RuntimeException("Unable to copy memstore timestamp " +
304 memstoreTS + " after decoding a key/value");
305 }
306 }
307 }
308
309 @Override
310 public HFileBlockEncodingContext newDataBlockEncodingContext(
311 Algorithm compressionAlgorithm,
312 DataBlockEncoding encoding, byte[] header) {
313 return new HFileBlockDefaultEncodingContext(
314 compressionAlgorithm, encoding, header);
315 }
316
317 @Override
318 public HFileBlockDecodingContext newDataBlockDecodingContext(
319 Algorithm compressionAlgorithm) {
320 return new HFileBlockDefaultDecodingContext(compressionAlgorithm);
321 }
322
323
324
325
326
327
328
329
330
331 public abstract void internalEncodeKeyValues(DataOutputStream out,
332 ByteBuffer in, boolean includesMemstoreTS) throws IOException;
333
334 @Override
335 public void encodeKeyValues(ByteBuffer in,
336 boolean includesMemstoreTS,
337 HFileBlockEncodingContext blkEncodingCtx) throws IOException {
338 if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
339 throw new IOException (this.getClass().getName() + " only accepts "
340 + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
341 "encoding context.");
342 }
343
344 HFileBlockDefaultEncodingContext encodingCtx =
345 (HFileBlockDefaultEncodingContext) blkEncodingCtx;
346 encodingCtx.prepareEncoding();
347 DataOutputStream dataOut =
348 ((HFileBlockDefaultEncodingContext) encodingCtx)
349 .getOutputStreamForEncoder();
350 internalEncodeKeyValues(dataOut, in, includesMemstoreTS);
351 if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
352 encodingCtx.postEncoding(BlockType.ENCODED_DATA);
353 } else {
354 encodingCtx.postEncoding(BlockType.DATA);
355 }
356 }
357
358
359
360
361
362
363
364
365 protected static void ensureSpace(ByteBuffer out, int length)
366 throws EncoderBufferTooSmallException {
367 if (out.position() + length > out.limit()) {
368 throw new EncoderBufferTooSmallException(
369 "Buffer position=" + out.position() +
370 ", buffer limit=" + out.limit() +
371 ", length to be written=" + length);
372 }
373 }
374
375 }