View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  
25  import org.apache.hadoop.hbase.KeyValue;
26  import org.apache.hadoop.hbase.util.Bytes;
27  import org.apache.hadoop.io.WritableUtils;
28  
29  /**
30   * Compression class for {@link KeyValue}s written to the WAL. This is not
31   * synchronized, so synchronization should be handled outside.
32   * 
33   * Class only compresses and uncompresses row keys, family names, and the
34   * qualifier. More may be added depending on use patterns.
35   */
36  class KeyValueCompression {
37    /**
38     * Uncompresses a KeyValue from a DataInput and returns it.
39     * 
40     * @param in the DataInput
41     * @param readContext the compressionContext to use.
42     * @return an uncompressed KeyValue
43     * @throws IOException
44     */
45  
46    public static KeyValue readKV(DataInput in, CompressionContext readContext)
47        throws IOException {
48      int keylength = WritableUtils.readVInt(in);
49      int vlength = WritableUtils.readVInt(in);
50      int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
51  
52      byte[] backingArray = new byte[length];
53      int pos = 0;
54      pos = Bytes.putInt(backingArray, pos, keylength);
55      pos = Bytes.putInt(backingArray, pos, vlength);
56  
57      // the row
58      int elemLen = Compressor.uncompressIntoArray(backingArray,
59          pos + Bytes.SIZEOF_SHORT, in, readContext.rowDict);
60      checkLength(elemLen, Short.MAX_VALUE);
61      pos = Bytes.putShort(backingArray, pos, (short)elemLen);
62      pos += elemLen;
63  
64      // family
65      elemLen = Compressor.uncompressIntoArray(backingArray,
66          pos + Bytes.SIZEOF_BYTE, in, readContext.familyDict);
67      checkLength(elemLen, Byte.MAX_VALUE);
68      pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
69      pos += elemLen;
70  
71      // qualifier
72      elemLen = Compressor.uncompressIntoArray(backingArray, pos, in,
73          readContext.qualifierDict);
74      pos += elemLen;
75  
76      // the rest
77      in.readFully(backingArray, pos, length - pos);
78  
79      return new KeyValue(backingArray);
80    }
81  
82    private static void checkLength(int len, int max) throws IOException {
83      if (len < 0 || len > max) {
84        throw new IOException(
85            "Invalid length for compresesed portion of keyvalue: " + len);
86      }
87    }
88  
89    /**
90     * Compresses and writes ourKV to out, a DataOutput.
91     * 
92     * @param out the DataOutput
93     * @param keyVal the KV to compress and write
94     * @param writeContext the compressionContext to use.
95     * @throws IOException
96     */
97    public static void writeKV(final DataOutput out, KeyValue keyVal,
98        CompressionContext writeContext) throws IOException {
99      byte[] backingArray = keyVal.getBuffer();
100     int offset = keyVal.getOffset();
101 
102     // we first write the KeyValue infrastructure as VInts.
103     WritableUtils.writeVInt(out, keyVal.getKeyLength());
104     WritableUtils.writeVInt(out, keyVal.getValueLength());
105 
106     // now we write the row key, as the row key is likely to be repeated
107     // We save space only if we attempt to compress elements with duplicates
108     Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getRowOffset(),
109         keyVal.getRowLength(), out, writeContext.rowDict);
110 
111   
112     // now family, if it exists. if it doesn't, we write a 0 length array.
113     Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getFamilyOffset(),
114         keyVal.getFamilyLength(), out, writeContext.familyDict);
115 
116     // qualifier next
117     Compressor.writeCompressed(keyVal.getBuffer(), keyVal.getQualifierOffset(),
118         keyVal.getQualifierLength(), out,
119         writeContext.qualifierDict);
120 
121     // now we write the rest uncompressed
122     int pos = keyVal.getTimestampOffset();
123     int remainingLength = keyVal.getLength() + offset - (pos);
124     out.write(backingArray, pos, remainingLength);
125   }
126 }