View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver.wal;
19  
20  import java.io.ByteArrayOutputStream;
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.hbase.Cell;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.codec.BaseDecoder;
29  import org.apache.hadoop.hbase.codec.BaseEncoder;
30  import org.apache.hadoop.hbase.codec.Codec;
31  import org.apache.hadoop.hbase.codec.KeyValueCodec;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.apache.hadoop.hbase.util.ReflectionUtils;
34  import org.apache.hadoop.io.IOUtils;
35  
36  import com.google.common.base.Preconditions;
37  import com.google.protobuf.ByteString;
38  
39  
40  /**
41   * Compression in this class is lifted off Compressor/KeyValueCompression.
42   * This is a pure coincidence... they are independent and don't have to be compatible.
43   */
44  public class WALCellCodec implements Codec {
45    /** Configuration key for the class to use when encoding cells in the WAL */
46    public static final String WAL_CELL_CODEC_CLASS_KEY = "hbase.regionserver.wal.codec";
47  
48    private final CompressionContext compression;
49    private final ByteStringUncompressor statelessUncompressor = new ByteStringUncompressor() {
50      @Override
51      public byte[] uncompress(ByteString data, Dictionary dict) throws IOException {
52        return WALCellCodec.uncompressByteString(data, dict);
53      }
54    };
55  
56    /**
57     * Default constructor - <b>all subclasses must implement a constructor with this signature </b>
58     * if they are to be dynamically loaded from the {@link Configuration}.
59     * @param conf configuration to configure <tt>this</tt>
60     * @param compression compression the codec should support, can be <tt>null</tt> to indicate no
61     *          compression
62     */
63    public WALCellCodec(Configuration conf, CompressionContext compression) {
64      this.compression = compression;
65    }
66  
67    /**
68     * Create and setup a {@link WALCellCodec} from the {@link Configuration} and CompressionContext,
69     * if they have been specified. Fully prepares the codec for use.
70     * @param conf {@link Configuration} to read for the user-specified codec. If none is specified,
71     *          uses a {@link WALCellCodec}.
72     * @param compression compression the codec should use
73     * @return a {@link WALCellCodec} ready for use.
74     * @throws UnsupportedOperationException if the codec cannot be instantiated
75     */
76    public static WALCellCodec create(Configuration conf, CompressionContext compression)
77        throws UnsupportedOperationException {
78      String className = conf.get(WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
79      return ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class,
80          CompressionContext.class }, new Object[] { conf, compression });
81    }
82  
83    public interface ByteStringCompressor {
84      ByteString compress(byte[] data, Dictionary dict) throws IOException;
85    }
86  
87    public interface ByteStringUncompressor {
88      byte[] uncompress(ByteString data, Dictionary dict) throws IOException;
89    }
90  
91    // TODO: it sucks that compression context is in HLog.Entry. It'd be nice if it was here.
92    //       Dictionary could be gotten by enum; initially, based on enum, context would create
93    //       an array of dictionaries.
94    static class BaosAndCompressor extends ByteArrayOutputStream implements ByteStringCompressor {
95      public ByteString toByteString() {
96        return ByteString.copyFrom(this.buf, 0, this.count);
97      }
98  
99      @Override
100     public ByteString compress(byte[] data, Dictionary dict) throws IOException {
101       writeCompressed(data, dict);
102       ByteString result = ByteString.copyFrom(this.buf, 0, this.count);
103       reset(); // Only resets the count - we reuse the byte array.
104       return result;
105     }
106 
107     private void writeCompressed(byte[] data, Dictionary dict) throws IOException {
108       assert dict != null;
109       short dictIdx = dict.findEntry(data, 0, data.length);
110       if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
111         write(Dictionary.NOT_IN_DICTIONARY);
112         StreamUtils.writeRawVInt32(this, data.length);
113         write(data, 0, data.length);
114       } else {
115         StreamUtils.writeShort(this, dictIdx);
116       }
117     }
118   }
119 
120   private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
121     InputStream in = bs.newInput();
122     byte status = (byte)in.read();
123     if (status == Dictionary.NOT_IN_DICTIONARY) {
124       byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
125       int bytesRead = in.read(arr);
126       if (bytesRead != arr.length) {
127         throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
128       }
129       if (dict != null) dict.addEntry(arr, 0, arr.length);
130       return arr;
131     } else {
132       // Status here is the higher-order byte of index of the dictionary entry.
133       short dictIdx = StreamUtils.toShort(status, (byte)in.read());
134       byte[] entry = dict.getEntry(dictIdx);
135       if (entry == null) {
136         throw new IOException("Missing dictionary entry for index " + dictIdx);
137       }
138       return entry;
139     }
140   }
141 
142   static class CompressedKvEncoder extends BaseEncoder {
143     private final CompressionContext compression;
144     public CompressedKvEncoder(OutputStream out, CompressionContext compression) {
145       super(out);
146       this.compression = compression;
147     }
148 
149     @Override
150     public void write(Cell cell) throws IOException {
151       if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
152       KeyValue kv = (KeyValue)cell;
153       byte[] kvBuffer = kv.getBuffer();
154       int offset = kv.getOffset();
155 
156       // We first write the KeyValue infrastructure as VInts.
157       StreamUtils.writeRawVInt32(out, kv.getKeyLength());
158       StreamUtils.writeRawVInt32(out, kv.getValueLength());
159 
160       // Write row, qualifier, and family; use dictionary
161       // compression as they're likely to have duplicates.
162       write(kvBuffer, kv.getRowOffset(), kv.getRowLength(), compression.rowDict);
163       write(kvBuffer, kv.getFamilyOffset(), kv.getFamilyLength(), compression.familyDict);
164       write(kvBuffer, kv.getQualifierOffset(), kv.getQualifierLength(), compression.qualifierDict);
165 
166       // Write the rest uncompressed.
167       int pos = kv.getTimestampOffset();
168       int remainingLength = kv.getLength() + offset - pos;
169       out.write(kvBuffer, pos, remainingLength);
170 
171     }
172 
173     private void write(byte[] data, int offset, int length, Dictionary dict) throws IOException {
174       short dictIdx = Dictionary.NOT_IN_DICTIONARY;
175       if (dict != null) {
176         dictIdx = dict.findEntry(data, offset, length);
177       }
178       if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
179         out.write(Dictionary.NOT_IN_DICTIONARY);
180         StreamUtils.writeRawVInt32(out, length);
181         out.write(data, offset, length);
182       } else {
183         StreamUtils.writeShort(out, dictIdx);
184       }
185     }
186   }
187 
188   static class CompressedKvDecoder extends BaseDecoder {
189     private final CompressionContext compression;
190     public CompressedKvDecoder(InputStream in, CompressionContext compression) {
191       super(in);
192       this.compression = compression;
193     }
194 
195     @Override
196     protected Cell parseCell() throws IOException {
197       int keylength = StreamUtils.readRawVarint32(in);
198       int vlength = StreamUtils.readRawVarint32(in);
199       int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
200 
201       byte[] backingArray = new byte[length];
202       int pos = 0;
203       pos = Bytes.putInt(backingArray, pos, keylength);
204       pos = Bytes.putInt(backingArray, pos, vlength);
205 
206       // the row
207       int elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_SHORT, compression.rowDict);
208       checkLength(elemLen, Short.MAX_VALUE);
209       pos = Bytes.putShort(backingArray, pos, (short)elemLen);
210       pos += elemLen;
211 
212       // family
213       elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_BYTE, compression.familyDict);
214       checkLength(elemLen, Byte.MAX_VALUE);
215       pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
216       pos += elemLen;
217 
218       // qualifier
219       elemLen = readIntoArray(backingArray, pos, compression.qualifierDict);
220       pos += elemLen;
221 
222       // the rest
223       IOUtils.readFully(in, backingArray, pos, length - pos);
224       return new KeyValue(backingArray);
225     }
226 
227     private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
228       byte status = (byte)in.read();
229       if (status == Dictionary.NOT_IN_DICTIONARY) {
230         // status byte indicating that data to be read is not in dictionary.
231         // if this isn't in the dictionary, we need to add to the dictionary.
232         int length = StreamUtils.readRawVarint32(in);
233         IOUtils.readFully(in, to, offset, length);
234         dict.addEntry(to, offset, length);
235         return length;
236       } else {
237         // the status byte also acts as the higher order byte of the dictionary entry.
238         short dictIdx = StreamUtils.toShort(status, (byte)in.read());
239         byte[] entry = dict.getEntry(dictIdx);
240         if (entry == null) {
241           throw new IOException("Missing dictionary entry for index " + dictIdx);
242         }
243         // now we write the uncompressed value.
244         Bytes.putBytes(to, offset, entry, 0, entry.length);
245         return entry.length;
246       }
247     }
248 
249     private static void checkLength(int len, int max) throws IOException {
250       if (len < 0 || len > max) {
251         throw new IOException("Invalid length for compresesed portion of keyvalue: " + len);
252       }
253     }
254   }
255 
256   public class EnsureKvEncoder extends KeyValueCodec.KeyValueEncoder {
257     public EnsureKvEncoder(OutputStream out) {
258       super(out);
259     }
260     @Override
261     public void write(Cell cell) throws IOException {
262       if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
263       super.write(cell);
264     }
265   }
266 
267   @Override
268   public Decoder getDecoder(InputStream is) {
269     return (compression == null)
270         ? new KeyValueCodec.KeyValueDecoder(is) : new CompressedKvDecoder(is, compression);
271   }
272 
273   @Override
274   public Encoder getEncoder(OutputStream os) {
275     return (compression == null)
276         ? new EnsureKvEncoder(os) : new CompressedKvEncoder(os, compression);
277   }
278 
279   public ByteStringCompressor getByteStringCompressor() {
280     // TODO: ideally this should also encapsulate compressionContext
281     return new BaosAndCompressor();
282   }
283 
284   public ByteStringUncompressor getByteStringUncompressor() {
285     // TODO: ideally this should also encapsulate compressionContext
286     return this.statelessUncompressor;
287   }
288 
289   /**
290    * It seems like as soon as somebody sets himself to the task of creating VInt encoding,
291    * his mind blanks out for a split-second and he starts the work by wrapping it in the
292    * most convoluted interface he can come up with. Custom streams that allocate memory,
293    * DataOutput that is only used to write single bytes... We operate on simple streams.
294    * Thus, we are going to have a simple implementation copy-pasted from protobuf Coded*Stream.
295    */
296   private static class StreamUtils {
297     public static int computeRawVarint32Size(final int value) {
298       if ((value & (0xffffffff <<  7)) == 0) return 1;
299       if ((value & (0xffffffff << 14)) == 0) return 2;
300       if ((value & (0xffffffff << 21)) == 0) return 3;
301       if ((value & (0xffffffff << 28)) == 0) return 4;
302       return 5;
303     }
304 
305     static void writeRawVInt32(OutputStream output, int value) throws IOException {
306       assert value >= 0;
307       while (true) {
308         if ((value & ~0x7F) == 0) {
309           output.write(value);
310           return;
311         } else {
312           output.write((value & 0x7F) | 0x80);
313           value >>>= 7;
314         }
315       }
316     }
317 
318     static int readRawVarint32(InputStream input) throws IOException {
319       byte tmp = (byte)input.read();
320       if (tmp >= 0) {
321         return tmp;
322       }
323       int result = tmp & 0x7f;
324       if ((tmp = (byte)input.read()) >= 0) {
325         result |= tmp << 7;
326       } else {
327         result |= (tmp & 0x7f) << 7;
328         if ((tmp = (byte)input.read()) >= 0) {
329           result |= tmp << 14;
330         } else {
331           result |= (tmp & 0x7f) << 14;
332           if ((tmp = (byte)input.read()) >= 0) {
333             result |= tmp << 21;
334           } else {
335             result |= (tmp & 0x7f) << 21;
336             result |= (tmp = (byte)input.read()) << 28;
337             if (tmp < 0) {
338               // Discard upper 32 bits.
339               for (int i = 0; i < 5; i++) {
340                 if (input.read() >= 0) {
341                   return result;
342                 }
343               }
344               throw new IOException("Malformed varint");
345             }
346           }
347         }
348       }
349       return result;
350     }
351 
352     static short toShort(byte hi, byte lo) {
353       short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
354       Preconditions.checkArgument(s >= 0);
355       return s;
356     }
357 
358     static void writeShort(OutputStream out, short v) throws IOException {
359       Preconditions.checkArgument(v >= 0);
360       out.write((byte)(0xff & (v >> 8)));
361       out.write((byte)(0xff & v));
362     }
363   }
364 }