View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License
17   */
18  
19  package org.apache.hadoop.hbase.regionserver.wal;
20  import java.io.DataInput;
21  import java.io.DataOutput;
22  import java.io.IOException;
23  
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.HBaseConfiguration;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.hadoop.io.WritableUtils;
32  
33  import com.google.common.base.Preconditions;
34  
35  /**
36   * A set of static functions for running our custom WAL compression/decompression.
37   * Also contains a command line tool to compress and uncompress HLogs.
38   */
39  @InterfaceAudience.Private
40  public class Compressor {
41    /**
42     * Command line tool to compress and uncompress WALs.
43     */
44    public static void main(String[] args) throws IOException {
45      if (args.length != 2 || args[0].equals("--help") || args[0].equals("-h")) {
46        printHelp();
47        System.exit(-1);
48      }
49  
50      Path inputPath = new Path(args[0]);
51      Path outputPath = new Path(args[1]);
52  
53      transformFile(inputPath, outputPath);
54    }
55  
56    private static void printHelp() {
57      System.err.println("usage: Compressor <input> <output>");
58      System.err.println("If <input> HLog is compressed, <output> will be decompressed.");
59      System.err.println("If <input> HLog is uncompressed, <output> will be compressed.");
60      return;
61    }
62  
63    private static void transformFile(Path input, Path output)
64        throws IOException {
65      Configuration conf = HBaseConfiguration.create();
66  
67      FileSystem inFS = input.getFileSystem(conf);
68      FileSystem outFS = output.getFileSystem(conf);
69  
70      HLog.Reader in = HLogFactory.createReader(inFS, input, conf, null, false);
71      HLog.Writer out = null;
72  
73      try {
74        if (!(in instanceof ReaderBase)) {
75          System.err.println("Cannot proceed, invalid reader type: " + in.getClass().getName());
76          return;
77        }
78        boolean compress = ((ReaderBase)in).hasCompression();
79        conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress);
80        out = HLogFactory.createWALWriter(outFS, output, conf);
81  
82        HLog.Entry e = null;
83        while ((e = in.next()) != null) out.append(e);
84      } finally {
85        in.close();
86        if (out != null) {
87          out.close();
88          out = null;
89        }
90      }
91    }
92  
93    /**
94     * Reads the next compressed entry and returns it as a byte array
95     * 
96     * @param in the DataInput to read from
97     * @param dict the dictionary we use for our read.
98     * @return the uncompressed array.
99     */
100   @Deprecated
101   static byte[] readCompressed(DataInput in, Dictionary dict)
102       throws IOException {
103     byte status = in.readByte();
104 
105     if (status == Dictionary.NOT_IN_DICTIONARY) {
106       int length = WritableUtils.readVInt(in);
107       // if this isn't in the dictionary, we need to add to the dictionary.
108       byte[] arr = new byte[length];
109       in.readFully(arr);
110       if (dict != null) dict.addEntry(arr, 0, length);
111       return arr;
112     } else {
113       // Status here is the higher-order byte of index of the dictionary entry
114       // (when its not Dictionary.NOT_IN_DICTIONARY -- dictionary indices are
115       // shorts).
116       short dictIdx = toShort(status, in.readByte());
117       byte[] entry = dict.getEntry(dictIdx);
118       if (entry == null) {
119         throw new IOException("Missing dictionary entry for index "
120             + dictIdx);
121       }
122       return entry;
123     }
124   }
125 
126   /**
127    * Reads a compressed entry into an array.
128    * The output into the array ends up length-prefixed.
129    * 
130    * @param to the array to write into
131    * @param offset array offset to start writing to
132    * @param in the DataInput to read from
133    * @param dict the dictionary to use for compression
134    * 
135    * @return the length of the uncompressed data
136    */
137   @Deprecated
138   static int uncompressIntoArray(byte[] to, int offset, DataInput in,
139       Dictionary dict) throws IOException {
140     byte status = in.readByte();
141 
142     if (status == Dictionary.NOT_IN_DICTIONARY) {
143       // status byte indicating that data to be read is not in dictionary.
144       // if this isn't in the dictionary, we need to add to the dictionary.
145       int length = WritableUtils.readVInt(in);
146       in.readFully(to, offset, length);
147       dict.addEntry(to, offset, length);
148       return length;
149     } else {
150       // the status byte also acts as the higher order byte of the dictionary
151       // entry
152       short dictIdx = toShort(status, in.readByte());
153       byte[] entry;
154       try {
155         entry = dict.getEntry(dictIdx);
156       } catch (Exception ex) {
157         throw new IOException("Unable to uncompress the log entry", ex);
158       }
159       if (entry == null) {
160         throw new IOException("Missing dictionary entry for index "
161             + dictIdx);
162       }
163       // now we write the uncompressed value.
164       Bytes.putBytes(to, offset, entry, 0, entry.length);
165       return entry.length;
166     }
167   }
168 
169   /**
170    * Compresses and writes an array to a DataOutput
171    * 
172    * @param data the array to write.
173    * @param out the DataOutput to write into
174    * @param dict the dictionary to use for compression
175    */
176   @Deprecated
177   static void writeCompressed(byte[] data, int offset, int length,
178       DataOutput out, Dictionary dict)
179       throws IOException {
180     short dictIdx = Dictionary.NOT_IN_DICTIONARY;
181     if (dict != null) {
182       dictIdx = dict.findEntry(data, offset, length);
183     }
184     if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
185       // not in dict
186       out.writeByte(Dictionary.NOT_IN_DICTIONARY);
187       WritableUtils.writeVInt(out, length);
188       out.write(data, offset, length);
189     } else {
190       out.writeShort(dictIdx);
191     }
192   }
193 
194   static short toShort(byte hi, byte lo) {
195     short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
196     Preconditions.checkArgument(s >= 0);
197     return s;
198   }
199 }