1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.wal;
19
20 import java.io.ByteArrayOutputStream;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.hbase.Cell;
27 import org.apache.hadoop.hbase.KeyValue;
28 import org.apache.hadoop.hbase.codec.BaseDecoder;
29 import org.apache.hadoop.hbase.codec.BaseEncoder;
30 import org.apache.hadoop.hbase.codec.Codec;
31 import org.apache.hadoop.hbase.codec.KeyValueCodec;
32 import org.apache.hadoop.hbase.util.Bytes;
33 import org.apache.hadoop.hbase.util.ReflectionUtils;
34 import org.apache.hadoop.io.IOUtils;
35
36 import com.google.common.base.Preconditions;
37 import com.google.protobuf.ByteString;
38
39
40
41
42
43
44 public class WALCellCodec implements Codec {
45
46 public static final String WAL_CELL_CODEC_CLASS_KEY = "hbase.regionserver.wal.codec";
47
48 private final CompressionContext compression;
49 private final ByteStringUncompressor statelessUncompressor = new ByteStringUncompressor() {
50 @Override
51 public byte[] uncompress(ByteString data, Dictionary dict) throws IOException {
52 return WALCellCodec.uncompressByteString(data, dict);
53 }
54 };
55
56
57
58
59
60
61
62
63 public WALCellCodec(Configuration conf, CompressionContext compression) {
64 this.compression = compression;
65 }
66
67
68
69
70
71
72
73
74
75
76 public static WALCellCodec create(Configuration conf, CompressionContext compression)
77 throws UnsupportedOperationException {
78 String className = conf.get(WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
79 return ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class,
80 CompressionContext.class }, new Object[] { conf, compression });
81 }
82
83 public interface ByteStringCompressor {
84 ByteString compress(byte[] data, Dictionary dict) throws IOException;
85 }
86
87 public interface ByteStringUncompressor {
88 byte[] uncompress(ByteString data, Dictionary dict) throws IOException;
89 }
90
91
92
93
94 static class BaosAndCompressor extends ByteArrayOutputStream implements ByteStringCompressor {
95 public ByteString toByteString() {
96 return ByteString.copyFrom(this.buf, 0, this.count);
97 }
98
99 @Override
100 public ByteString compress(byte[] data, Dictionary dict) throws IOException {
101 writeCompressed(data, dict);
102 ByteString result = ByteString.copyFrom(this.buf, 0, this.count);
103 reset();
104 return result;
105 }
106
107 private void writeCompressed(byte[] data, Dictionary dict) throws IOException {
108 assert dict != null;
109 short dictIdx = dict.findEntry(data, 0, data.length);
110 if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
111 write(Dictionary.NOT_IN_DICTIONARY);
112 StreamUtils.writeRawVInt32(this, data.length);
113 write(data, 0, data.length);
114 } else {
115 StreamUtils.writeShort(this, dictIdx);
116 }
117 }
118 }
119
120 private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
121 InputStream in = bs.newInput();
122 byte status = (byte)in.read();
123 if (status == Dictionary.NOT_IN_DICTIONARY) {
124 byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
125 int bytesRead = in.read(arr);
126 if (bytesRead != arr.length) {
127 throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
128 }
129 if (dict != null) dict.addEntry(arr, 0, arr.length);
130 return arr;
131 } else {
132
133 short dictIdx = StreamUtils.toShort(status, (byte)in.read());
134 byte[] entry = dict.getEntry(dictIdx);
135 if (entry == null) {
136 throw new IOException("Missing dictionary entry for index " + dictIdx);
137 }
138 return entry;
139 }
140 }
141
142 static class CompressedKvEncoder extends BaseEncoder {
143 private final CompressionContext compression;
144 public CompressedKvEncoder(OutputStream out, CompressionContext compression) {
145 super(out);
146 this.compression = compression;
147 }
148
149 @Override
150 public void write(Cell cell) throws IOException {
151 if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
152 KeyValue kv = (KeyValue)cell;
153 byte[] kvBuffer = kv.getBuffer();
154 int offset = kv.getOffset();
155
156
157 StreamUtils.writeRawVInt32(out, kv.getKeyLength());
158 StreamUtils.writeRawVInt32(out, kv.getValueLength());
159
160
161
162 write(kvBuffer, kv.getRowOffset(), kv.getRowLength(), compression.rowDict);
163 write(kvBuffer, kv.getFamilyOffset(), kv.getFamilyLength(), compression.familyDict);
164 write(kvBuffer, kv.getQualifierOffset(), kv.getQualifierLength(), compression.qualifierDict);
165
166
167 int pos = kv.getTimestampOffset();
168 int remainingLength = kv.getLength() + offset - pos;
169 out.write(kvBuffer, pos, remainingLength);
170
171 }
172
173 private void write(byte[] data, int offset, int length, Dictionary dict) throws IOException {
174 short dictIdx = Dictionary.NOT_IN_DICTIONARY;
175 if (dict != null) {
176 dictIdx = dict.findEntry(data, offset, length);
177 }
178 if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
179 out.write(Dictionary.NOT_IN_DICTIONARY);
180 StreamUtils.writeRawVInt32(out, length);
181 out.write(data, offset, length);
182 } else {
183 StreamUtils.writeShort(out, dictIdx);
184 }
185 }
186 }
187
188 static class CompressedKvDecoder extends BaseDecoder {
189 private final CompressionContext compression;
190 public CompressedKvDecoder(InputStream in, CompressionContext compression) {
191 super(in);
192 this.compression = compression;
193 }
194
195 @Override
196 protected Cell parseCell() throws IOException {
197 int keylength = StreamUtils.readRawVarint32(in);
198 int vlength = StreamUtils.readRawVarint32(in);
199 int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
200
201 byte[] backingArray = new byte[length];
202 int pos = 0;
203 pos = Bytes.putInt(backingArray, pos, keylength);
204 pos = Bytes.putInt(backingArray, pos, vlength);
205
206
207 int elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_SHORT, compression.rowDict);
208 checkLength(elemLen, Short.MAX_VALUE);
209 pos = Bytes.putShort(backingArray, pos, (short)elemLen);
210 pos += elemLen;
211
212
213 elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_BYTE, compression.familyDict);
214 checkLength(elemLen, Byte.MAX_VALUE);
215 pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
216 pos += elemLen;
217
218
219 elemLen = readIntoArray(backingArray, pos, compression.qualifierDict);
220 pos += elemLen;
221
222
223 IOUtils.readFully(in, backingArray, pos, length - pos);
224 return new KeyValue(backingArray);
225 }
226
227 private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
228 byte status = (byte)in.read();
229 if (status == Dictionary.NOT_IN_DICTIONARY) {
230
231
232 int length = StreamUtils.readRawVarint32(in);
233 IOUtils.readFully(in, to, offset, length);
234 dict.addEntry(to, offset, length);
235 return length;
236 } else {
237
238 short dictIdx = StreamUtils.toShort(status, (byte)in.read());
239 byte[] entry = dict.getEntry(dictIdx);
240 if (entry == null) {
241 throw new IOException("Missing dictionary entry for index " + dictIdx);
242 }
243
244 Bytes.putBytes(to, offset, entry, 0, entry.length);
245 return entry.length;
246 }
247 }
248
249 private static void checkLength(int len, int max) throws IOException {
250 if (len < 0 || len > max) {
251 throw new IOException("Invalid length for compresesed portion of keyvalue: " + len);
252 }
253 }
254 }
255
256 public class EnsureKvEncoder extends KeyValueCodec.KeyValueEncoder {
257 public EnsureKvEncoder(OutputStream out) {
258 super(out);
259 }
260 @Override
261 public void write(Cell cell) throws IOException {
262 if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
263 super.write(cell);
264 }
265 }
266
267 @Override
268 public Decoder getDecoder(InputStream is) {
269 return (compression == null)
270 ? new KeyValueCodec.KeyValueDecoder(is) : new CompressedKvDecoder(is, compression);
271 }
272
273 @Override
274 public Encoder getEncoder(OutputStream os) {
275 return (compression == null)
276 ? new EnsureKvEncoder(os) : new CompressedKvEncoder(os, compression);
277 }
278
279 public ByteStringCompressor getByteStringCompressor() {
280
281 return new BaosAndCompressor();
282 }
283
284 public ByteStringUncompressor getByteStringUncompressor() {
285
286 return this.statelessUncompressor;
287 }
288
289
290
291
292
293
294
295
296 private static class StreamUtils {
297 public static int computeRawVarint32Size(final int value) {
298 if ((value & (0xffffffff << 7)) == 0) return 1;
299 if ((value & (0xffffffff << 14)) == 0) return 2;
300 if ((value & (0xffffffff << 21)) == 0) return 3;
301 if ((value & (0xffffffff << 28)) == 0) return 4;
302 return 5;
303 }
304
305 static void writeRawVInt32(OutputStream output, int value) throws IOException {
306 assert value >= 0;
307 while (true) {
308 if ((value & ~0x7F) == 0) {
309 output.write(value);
310 return;
311 } else {
312 output.write((value & 0x7F) | 0x80);
313 value >>>= 7;
314 }
315 }
316 }
317
318 static int readRawVarint32(InputStream input) throws IOException {
319 byte tmp = (byte)input.read();
320 if (tmp >= 0) {
321 return tmp;
322 }
323 int result = tmp & 0x7f;
324 if ((tmp = (byte)input.read()) >= 0) {
325 result |= tmp << 7;
326 } else {
327 result |= (tmp & 0x7f) << 7;
328 if ((tmp = (byte)input.read()) >= 0) {
329 result |= tmp << 14;
330 } else {
331 result |= (tmp & 0x7f) << 14;
332 if ((tmp = (byte)input.read()) >= 0) {
333 result |= tmp << 21;
334 } else {
335 result |= (tmp & 0x7f) << 21;
336 result |= (tmp = (byte)input.read()) << 28;
337 if (tmp < 0) {
338
339 for (int i = 0; i < 5; i++) {
340 if (input.read() >= 0) {
341 return result;
342 }
343 }
344 throw new IOException("Malformed varint");
345 }
346 }
347 }
348 }
349 return result;
350 }
351
352 static short toShort(byte hi, byte lo) {
353 short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
354 Preconditions.checkArgument(s >= 0);
355 return s;
356 }
357
358 static void writeShort(OutputStream out, short v) throws IOException {
359 Preconditions.checkArgument(v >= 0);
360 out.write((byte)(0xff & (v >> 8)));
361 out.write((byte)(0xff & v));
362 }
363 }
364 }