1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.wal;
19
20 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.hbase.Cell;
27 import org.apache.hadoop.hbase.KeyValue;
28 import org.apache.hadoop.hbase.codec.BaseDecoder;
29 import org.apache.hadoop.hbase.codec.BaseEncoder;
30 import org.apache.hadoop.hbase.codec.Codec;
31 import org.apache.hadoop.hbase.codec.KeyValueCodec;
32 import org.apache.hadoop.hbase.util.Bytes;
33 import org.apache.hadoop.hbase.util.ReflectionUtils;
34 import org.apache.hadoop.io.IOUtils;
35
36 import com.google.common.base.Preconditions;
37 import com.google.protobuf.ByteString;
38
39
40
41
42
43
44 public class WALCellCodec implements Codec {
45
46 public static final String WAL_CELL_CODEC_CLASS_KEY = "hbase.regionserver.wal.codec";
47
48 private final CompressionContext compression;
49 private final ByteStringUncompressor statelessUncompressor = new ByteStringUncompressor() {
50 @Override
51 public byte[] uncompress(ByteString data, Dictionary dict) throws IOException {
52 return WALCellCodec.uncompressByteString(data, dict);
53 }
54 };
55
56
57
58
59
60
61
62
63 public WALCellCodec(Configuration conf, CompressionContext compression) {
64 this.compression = compression;
65 }
66
67
68
69
70
71
72
73
74
75
76 public static WALCellCodec create(Configuration conf, CompressionContext compression)
77 throws UnsupportedOperationException {
78 String className = conf.get(WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
79 return ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class,
80 CompressionContext.class }, new Object[] { conf, compression });
81 }
82
83 public interface ByteStringCompressor {
84 ByteString compress(byte[] data, Dictionary dict) throws IOException;
85 }
86
87 public interface ByteStringUncompressor {
88 byte[] uncompress(ByteString data, Dictionary dict) throws IOException;
89 }
90
91
92
93
94 static class BaosAndCompressor extends ByteArrayOutputStream implements ByteStringCompressor {
95 public ByteString toByteString() {
96 return ByteString.copyFrom(this.buf, 0, this.count);
97 }
98
99 @Override
100 public ByteString compress(byte[] data, Dictionary dict) throws IOException {
101 writeCompressed(data, dict);
102 ByteString result = ByteString.copyFrom(this.buf, 0, this.count);
103 reset();
104 return result;
105 }
106
107 private void writeCompressed(byte[] data, Dictionary dict) throws IOException {
108 assert dict != null;
109 short dictIdx = dict.findEntry(data, 0, data.length);
110 if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
111 write(Dictionary.NOT_IN_DICTIONARY);
112 StreamUtils.writeRawVInt32(this, data.length);
113 write(data, 0, data.length);
114 } else {
115 StreamUtils.writeShort(this, dictIdx);
116 }
117 }
118 }
119
120 private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
121 InputStream in = bs.newInput();
122 byte status = (byte)in.read();
123 if (status == Dictionary.NOT_IN_DICTIONARY) {
124 byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
125 int bytesRead = in.read(arr);
126 if (bytesRead != arr.length) {
127 throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
128 }
129 if (dict != null) dict.addEntry(arr, 0, arr.length);
130 return arr;
131 } else {
132
133 short dictIdx = StreamUtils.toShort(status, (byte)in.read());
134 byte[] entry = dict.getEntry(dictIdx);
135 if (entry == null) {
136 throw new IOException("Missing dictionary entry for index " + dictIdx);
137 }
138 return entry;
139 }
140 }
141
142 static class CompressedKvEncoder extends BaseEncoder {
143 private final CompressionContext compression;
144 public CompressedKvEncoder(OutputStream out, CompressionContext compression) {
145 super(out);
146 this.compression = compression;
147 }
148
149 @Override
150 public void write(Cell cell) throws IOException {
151 if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
152 KeyValue kv = (KeyValue)cell;
153 byte[] kvBuffer = kv.getBuffer();
154 int offset = kv.getOffset();
155
156
157 StreamUtils.writeRawVInt32(out, kv.getKeyLength());
158 StreamUtils.writeRawVInt32(out, kv.getValueLength());
159
160 StreamUtils.writeRawVInt32(out, (short)0);
161
162
163
164 write(kvBuffer, kv.getRowOffset(), kv.getRowLength(), compression.rowDict);
165 write(kvBuffer, kv.getFamilyOffset(), kv.getFamilyLength(), compression.familyDict);
166 write(kvBuffer, kv.getQualifierOffset(), kv.getQualifierLength(), compression.qualifierDict);
167
168
169 int pos = kv.getTimestampOffset();
170 int remainingLength = kv.getLength() + offset - pos;
171 out.write(kvBuffer, pos, remainingLength);
172
173 }
174
175 private void write(byte[] data, int offset, int length, Dictionary dict) throws IOException {
176 short dictIdx = Dictionary.NOT_IN_DICTIONARY;
177 if (dict != null) {
178 dictIdx = dict.findEntry(data, offset, length);
179 }
180 if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
181 out.write(Dictionary.NOT_IN_DICTIONARY);
182 StreamUtils.writeRawVInt32(out, length);
183 out.write(data, offset, length);
184 } else {
185 StreamUtils.writeShort(out, dictIdx);
186 }
187 }
188 }
189
190 static class CompressedKvDecoder extends BaseDecoder {
191 private final CompressionContext compression;
192 public CompressedKvDecoder(InputStream in, CompressionContext compression) {
193 super(in);
194 this.compression = compression;
195 }
196
197 @Override
198 protected Cell parseCell() throws IOException {
199 int keylength = StreamUtils.readRawVarint32(in);
200 int vlength = StreamUtils.readRawVarint32(in);
201
202
203
204 StreamUtils.readRawVarint32(in);
205 int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
206
207 byte[] backingArray = new byte[length];
208 int pos = 0;
209 pos = Bytes.putInt(backingArray, pos, keylength);
210 pos = Bytes.putInt(backingArray, pos, vlength);
211
212
213 int elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_SHORT, compression.rowDict);
214 checkLength(elemLen, Short.MAX_VALUE);
215 pos = Bytes.putShort(backingArray, pos, (short)elemLen);
216 pos += elemLen;
217
218
219 elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_BYTE, compression.familyDict);
220 checkLength(elemLen, Byte.MAX_VALUE);
221 pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
222 pos += elemLen;
223
224
225 elemLen = readIntoArray(backingArray, pos, compression.qualifierDict);
226 pos += elemLen;
227
228
229 IOUtils.readFully(in, backingArray, pos, length - pos);
230 return new KeyValue(backingArray, 0, length);
231 }
232
233 private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
234 byte status = (byte)in.read();
235 if (status == Dictionary.NOT_IN_DICTIONARY) {
236
237
238 int length = StreamUtils.readRawVarint32(in);
239 IOUtils.readFully(in, to, offset, length);
240 dict.addEntry(to, offset, length);
241 return length;
242 } else {
243
244 short dictIdx = StreamUtils.toShort(status, (byte)in.read());
245 byte[] entry = dict.getEntry(dictIdx);
246 if (entry == null) {
247 throw new IOException("Missing dictionary entry for index " + dictIdx);
248 }
249
250 Bytes.putBytes(to, offset, entry, 0, entry.length);
251 return entry.length;
252 }
253 }
254
255 private static void checkLength(int len, int max) throws IOException {
256 if (len < 0 || len > max) {
257 throw new IOException("Invalid length for compresesed portion of keyvalue: " + len);
258 }
259 }
260 }
261
262 public class EnsureKvEncoder extends KeyValueCodec.KeyValueEncoder {
263 public EnsureKvEncoder(OutputStream out) {
264 super(out);
265 }
266 @Override
267 public void write(Cell cell) throws IOException {
268 if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
269 super.write(cell);
270 }
271 }
272
273 @Override
274 public Decoder getDecoder(InputStream is) {
275 return (compression == null)
276 ? new KeyValueCodec.KeyValueDecoder(is) : new CompressedKvDecoder(is, compression);
277 }
278
279 @Override
280 public Encoder getEncoder(OutputStream os) {
281 return (compression == null)
282 ? new EnsureKvEncoder(os) : new CompressedKvEncoder(os, compression);
283 }
284
285 public ByteStringCompressor getByteStringCompressor() {
286
287 return new BaosAndCompressor();
288 }
289
290 public ByteStringUncompressor getByteStringUncompressor() {
291
292 return this.statelessUncompressor;
293 }
294
295
296
297
298
299
300
301
302 private static class StreamUtils {
303 public static int computeRawVarint32Size(final int value) {
304 if ((value & (0xffffffff << 7)) == 0) return 1;
305 if ((value & (0xffffffff << 14)) == 0) return 2;
306 if ((value & (0xffffffff << 21)) == 0) return 3;
307 if ((value & (0xffffffff << 28)) == 0) return 4;
308 return 5;
309 }
310
311 static void writeRawVInt32(OutputStream output, int value) throws IOException {
312 assert value >= 0;
313 while (true) {
314 if ((value & ~0x7F) == 0) {
315 output.write(value);
316 return;
317 } else {
318 output.write((value & 0x7F) | 0x80);
319 value >>>= 7;
320 }
321 }
322 }
323
324 static int readRawVarint32(InputStream input) throws IOException {
325 byte tmp = (byte)input.read();
326 if (tmp >= 0) {
327 return tmp;
328 }
329 int result = tmp & 0x7f;
330 if ((tmp = (byte)input.read()) >= 0) {
331 result |= tmp << 7;
332 } else {
333 result |= (tmp & 0x7f) << 7;
334 if ((tmp = (byte)input.read()) >= 0) {
335 result |= tmp << 14;
336 } else {
337 result |= (tmp & 0x7f) << 14;
338 if ((tmp = (byte)input.read()) >= 0) {
339 result |= tmp << 21;
340 } else {
341 result |= (tmp & 0x7f) << 21;
342 result |= (tmp = (byte)input.read()) << 28;
343 if (tmp < 0) {
344
345 for (int i = 0; i < 5; i++) {
346 if (input.read() >= 0) {
347 return result;
348 }
349 }
350 throw new IOException("Malformed varint");
351 }
352 }
353 }
354 }
355 return result;
356 }
357
358 static short toShort(byte hi, byte lo) {
359 short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
360 Preconditions.checkArgument(s >= 0);
361 return s;
362 }
363
364 static void writeShort(OutputStream out, short v) throws IOException {
365 Preconditions.checkArgument(v >= 0);
366 out.write((byte)(0xff & (v >> 8)));
367 out.write((byte)(0xff & v));
368 }
369 }
370 }