View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver.wal;
19  
20  import java.io.ByteArrayOutputStream;
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.hbase.Cell;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.codec.BaseDecoder;
29  import org.apache.hadoop.hbase.codec.BaseEncoder;
30  import org.apache.hadoop.hbase.codec.Codec;
31  import org.apache.hadoop.hbase.codec.KeyValueCodec;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.apache.hadoop.hbase.util.ReflectionUtils;
34  import org.apache.hadoop.io.IOUtils;
35  
36  import com.google.common.base.Preconditions;
37  import com.google.protobuf.ByteString;
38  
39  
40  /**
41   * Compression in this class is lifted off Compressor/KeyValueCompression.
42   * This is a pure coincidence... they are independent and don't have to be compatible.
43   */
44  public class WALCellCodec implements Codec {
45    /** Configuration key for the class to use when encoding cells in the WAL */
46    public static final String WAL_CELL_CODEC_CLASS_KEY = "hbase.regionserver.wal.codec";
47  
48    private final CompressionContext compression;
49    private final ByteStringUncompressor statelessUncompressor = new ByteStringUncompressor() {
50      @Override
51      public byte[] uncompress(ByteString data, Dictionary dict) throws IOException {
52        return WALCellCodec.uncompressByteString(data, dict);
53      }
54    };
55  
56    /**
57     * Default constructor - <b>all subclasses must implement a constructor with this signature </b>
58     * if they are to be dynamically loaded from the {@link Configuration}.
59     * @param conf configuration to configure <tt>this</tt>
60     * @param compression compression the codec should support, can be <tt>null</tt> to indicate no
61     *          compression
62     */
63    public WALCellCodec(Configuration conf, CompressionContext compression) {
64      this.compression = compression;
65    }
66  
67    /**
68     * Create and setup a {@link WALCellCodec} from the {@link Configuration} and CompressionContext,
69     * if they have been specified. Fully prepares the codec for use.
70     * @param conf {@link Configuration} to read for the user-specified codec. If none is specified,
71     *          uses a {@link WALCellCodec}.
72     * @param compression compression the codec should use
73     * @return a {@link WALCellCodec} ready for use.
74     * @throws UnsupportedOperationException if the codec cannot be instantiated
75     */
76    public static WALCellCodec create(Configuration conf, CompressionContext compression)
77        throws UnsupportedOperationException {
78      String className = conf.get(WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
79      return ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class,
80          CompressionContext.class }, new Object[] { conf, compression });
81    }
82  
83    public interface ByteStringCompressor {
84      ByteString compress(byte[] data, Dictionary dict) throws IOException;
85    }
86  
87    public interface ByteStringUncompressor {
88      byte[] uncompress(ByteString data, Dictionary dict) throws IOException;
89    }
90  
91    // TODO: it sucks that compression context is in HLog.Entry. It'd be nice if it was here.
92    //       Dictionary could be gotten by enum; initially, based on enum, context would create
93    //       an array of dictionaries.
94    static class BaosAndCompressor extends ByteArrayOutputStream implements ByteStringCompressor {
95      public ByteString toByteString() {
96        return ByteString.copyFrom(this.buf, 0, this.count);
97      }
98  
99      @Override
100     public ByteString compress(byte[] data, Dictionary dict) throws IOException {
101       writeCompressed(data, dict);
102       ByteString result = ByteString.copyFrom(this.buf, 0, this.count);
103       reset(); // Only resets the count - we reuse the byte array.
104       return result;
105     }
106 
107     private void writeCompressed(byte[] data, Dictionary dict) throws IOException {
108       assert dict != null;
109       short dictIdx = dict.findEntry(data, 0, data.length);
110       if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
111         write(Dictionary.NOT_IN_DICTIONARY);
112         StreamUtils.writeRawVInt32(this, data.length);
113         write(data, 0, data.length);
114       } else {
115         StreamUtils.writeShort(this, dictIdx);
116       }
117     }
118   }
119 
120   private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
121     InputStream in = bs.newInput();
122     byte status = (byte)in.read();
123     if (status == Dictionary.NOT_IN_DICTIONARY) {
124       byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
125       int bytesRead = in.read(arr);
126       if (bytesRead != arr.length) {
127         throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
128       }
129       if (dict != null) dict.addEntry(arr, 0, arr.length);
130       return arr;
131     } else {
132       // Status here is the higher-order byte of index of the dictionary entry.
133       short dictIdx = StreamUtils.toShort(status, (byte)in.read());
134       byte[] entry = dict.getEntry(dictIdx);
135       if (entry == null) {
136         throw new IOException("Missing dictionary entry for index " + dictIdx);
137       }
138       return entry;
139     }
140   }
141 
142   static class CompressedKvEncoder extends BaseEncoder {
143     private final CompressionContext compression;
144     public CompressedKvEncoder(OutputStream out, CompressionContext compression) {
145       super(out);
146       this.compression = compression;
147     }
148 
149     @Override
150     public void write(Cell cell) throws IOException {
151       if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
152       KeyValue kv = (KeyValue)cell;
153       byte[] kvBuffer = kv.getBuffer();
154       int offset = kv.getOffset();
155 
156       // We first write the KeyValue infrastructure as VInts.
157       StreamUtils.writeRawVInt32(out, kv.getKeyLength());
158       StreamUtils.writeRawVInt32(out, kv.getValueLength());
159       // To support tags. This will be replaced with kv.getTagsLength
160       StreamUtils.writeRawVInt32(out, (short)0);
161 
162       // Write row, qualifier, and family; use dictionary
163       // compression as they're likely to have duplicates.
164       write(kvBuffer, kv.getRowOffset(), kv.getRowLength(), compression.rowDict);
165       write(kvBuffer, kv.getFamilyOffset(), kv.getFamilyLength(), compression.familyDict);
166       write(kvBuffer, kv.getQualifierOffset(), kv.getQualifierLength(), compression.qualifierDict);
167 
168       // Write the rest uncompressed.
169       int pos = kv.getTimestampOffset();
170       int remainingLength = kv.getLength() + offset - pos;
171       out.write(kvBuffer, pos, remainingLength);
172 
173     }
174 
175     private void write(byte[] data, int offset, int length, Dictionary dict) throws IOException {
176       short dictIdx = Dictionary.NOT_IN_DICTIONARY;
177       if (dict != null) {
178         dictIdx = dict.findEntry(data, offset, length);
179       }
180       if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
181         out.write(Dictionary.NOT_IN_DICTIONARY);
182         StreamUtils.writeRawVInt32(out, length);
183         out.write(data, offset, length);
184       } else {
185         StreamUtils.writeShort(out, dictIdx);
186       }
187     }
188   }
189 
190   static class CompressedKvDecoder extends BaseDecoder {
191     private final CompressionContext compression;
192     public CompressedKvDecoder(InputStream in, CompressionContext compression) {
193       super(in);
194       this.compression = compression;
195     }
196 
197     @Override
198     protected Cell parseCell() throws IOException {
199       int keylength = StreamUtils.readRawVarint32(in);
200       int vlength = StreamUtils.readRawVarint32(in);
201       
202       // To support Tags..Tags length will be 0.
203       // For now ignore the read value. This will be the tagslength
204       StreamUtils.readRawVarint32(in);
205       int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
206 
207       byte[] backingArray = new byte[length];
208       int pos = 0;
209       pos = Bytes.putInt(backingArray, pos, keylength);
210       pos = Bytes.putInt(backingArray, pos, vlength);
211 
212       // the row
213       int elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_SHORT, compression.rowDict);
214       checkLength(elemLen, Short.MAX_VALUE);
215       pos = Bytes.putShort(backingArray, pos, (short)elemLen);
216       pos += elemLen;
217 
218       // family
219       elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_BYTE, compression.familyDict);
220       checkLength(elemLen, Byte.MAX_VALUE);
221       pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
222       pos += elemLen;
223 
224       // qualifier
225       elemLen = readIntoArray(backingArray, pos, compression.qualifierDict);
226       pos += elemLen;
227 
228       // the rest
229       IOUtils.readFully(in, backingArray, pos, length - pos);
230       return new KeyValue(backingArray, 0, length);
231     }
232 
233     private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
234       byte status = (byte)in.read();
235       if (status == Dictionary.NOT_IN_DICTIONARY) {
236         // status byte indicating that data to be read is not in dictionary.
237         // if this isn't in the dictionary, we need to add to the dictionary.
238         int length = StreamUtils.readRawVarint32(in);
239         IOUtils.readFully(in, to, offset, length);
240         dict.addEntry(to, offset, length);
241         return length;
242       } else {
243         // the status byte also acts as the higher order byte of the dictionary entry.
244         short dictIdx = StreamUtils.toShort(status, (byte)in.read());
245         byte[] entry = dict.getEntry(dictIdx);
246         if (entry == null) {
247           throw new IOException("Missing dictionary entry for index " + dictIdx);
248         }
249         // now we write the uncompressed value.
250         Bytes.putBytes(to, offset, entry, 0, entry.length);
251         return entry.length;
252       }
253     }
254 
255     private static void checkLength(int len, int max) throws IOException {
256       if (len < 0 || len > max) {
257         throw new IOException("Invalid length for compresesed portion of keyvalue: " + len);
258       }
259     }
260   }
261 
262   public class EnsureKvEncoder extends KeyValueCodec.KeyValueEncoder {
263     public EnsureKvEncoder(OutputStream out) {
264       super(out);
265     }
266     @Override
267     public void write(Cell cell) throws IOException {
268       if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
269       super.write(cell);
270     }
271   }
272 
273   @Override
274   public Decoder getDecoder(InputStream is) {
275     return (compression == null)
276         ? new KeyValueCodec.KeyValueDecoder(is) : new CompressedKvDecoder(is, compression);
277   }
278 
279   @Override
280   public Encoder getEncoder(OutputStream os) {
281     return (compression == null)
282         ? new EnsureKvEncoder(os) : new CompressedKvEncoder(os, compression);
283   }
284 
285   public ByteStringCompressor getByteStringCompressor() {
286     // TODO: ideally this should also encapsulate compressionContext
287     return new BaosAndCompressor();
288   }
289 
290   public ByteStringUncompressor getByteStringUncompressor() {
291     // TODO: ideally this should also encapsulate compressionContext
292     return this.statelessUncompressor;
293   }
294 
295   /**
296    * It seems like as soon as somebody sets himself to the task of creating VInt encoding,
297    * his mind blanks out for a split-second and he starts the work by wrapping it in the
298    * most convoluted interface he can come up with. Custom streams that allocate memory,
299    * DataOutput that is only used to write single bytes... We operate on simple streams.
300    * Thus, we are going to have a simple implementation copy-pasted from protobuf Coded*Stream.
301    */
302   private static class StreamUtils {
303     public static int computeRawVarint32Size(final int value) {
304       if ((value & (0xffffffff <<  7)) == 0) return 1;
305       if ((value & (0xffffffff << 14)) == 0) return 2;
306       if ((value & (0xffffffff << 21)) == 0) return 3;
307       if ((value & (0xffffffff << 28)) == 0) return 4;
308       return 5;
309     }
310 
311     static void writeRawVInt32(OutputStream output, int value) throws IOException {
312       assert value >= 0;
313       while (true) {
314         if ((value & ~0x7F) == 0) {
315           output.write(value);
316           return;
317         } else {
318           output.write((value & 0x7F) | 0x80);
319           value >>>= 7;
320         }
321       }
322     }
323 
324     static int readRawVarint32(InputStream input) throws IOException {
325       byte tmp = (byte)input.read();
326       if (tmp >= 0) {
327         return tmp;
328       }
329       int result = tmp & 0x7f;
330       if ((tmp = (byte)input.read()) >= 0) {
331         result |= tmp << 7;
332       } else {
333         result |= (tmp & 0x7f) << 7;
334         if ((tmp = (byte)input.read()) >= 0) {
335           result |= tmp << 14;
336         } else {
337           result |= (tmp & 0x7f) << 14;
338           if ((tmp = (byte)input.read()) >= 0) {
339             result |= tmp << 21;
340           } else {
341             result |= (tmp & 0x7f) << 21;
342             result |= (tmp = (byte)input.read()) << 28;
343             if (tmp < 0) {
344               // Discard upper 32 bits.
345               for (int i = 0; i < 5; i++) {
346                 if (input.read() >= 0) {
347                   return result;
348                 }
349               }
350               throw new IOException("Malformed varint");
351             }
352           }
353         }
354       }
355       return result;
356     }
357 
358     static short toShort(byte hi, byte lo) {
359       short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
360       Preconditions.checkArgument(s >= 0);
361       return s;
362     }
363 
364     static void writeShort(OutputStream out, short v) throws IOException {
365       Preconditions.checkArgument(v >= 0);
366       out.write((byte)(0xff & (v >> 8)));
367       out.write((byte)(0xff & v));
368     }
369   }
370 }