View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.encoding;
18  
19  import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.NONE;
20  
21  import java.io.ByteArrayOutputStream;
22  import java.io.DataOutputStream;
23  import java.io.IOException;
24  
25  import org.apache.hadoop.hbase.io.compress.Compression;
26  import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
27  import org.apache.hadoop.hbase.io.hfile.BlockType;
28  import org.apache.hadoop.io.compress.CompressionOutputStream;
29  import org.apache.hadoop.io.compress.Compressor;
30  
31  import com.google.common.base.Preconditions;
32  
33  /**
34   * A default implementation of {@link HFileBlockEncodingContext}. It will
35   * compress the data section as one continuous buffer.
36   *
37   * @see HFileBlockDefaultDecodingContext for the decompression part
38   *
39   */
40  public class HFileBlockDefaultEncodingContext implements
41      HFileBlockEncodingContext {
42  
43    private byte[] onDiskBytesWithHeader;
44    private byte[] uncompressedBytesWithHeader;
45    private BlockType blockType;
46    private final DataBlockEncoding encodingAlgo;
47  
48    /** Compressor, which is also reused between consecutive blocks. */
49    private Compressor compressor;
50  
51    /** Compression output stream */
52    private CompressionOutputStream compressionStream;
53  
54    /** Underlying stream to write compressed bytes to */
55    private ByteArrayOutputStream compressedByteStream;
56  
57    /** Compression algorithm for all blocks this instance writes. */
58    private final Compression.Algorithm compressionAlgorithm;
59  
60    private ByteArrayOutputStream encodedStream = new ByteArrayOutputStream();
61    private DataOutputStream dataOut = new DataOutputStream(encodedStream);
62  
63    private byte[] dummyHeader;
64  
65    /**
66     * @param compressionAlgorithm compression algorithm used
67     * @param encoding encoding used
68     * @param headerBytes dummy header bytes
69     */
70    public HFileBlockDefaultEncodingContext(
71        Compression.Algorithm compressionAlgorithm,
72        DataBlockEncoding encoding, byte[] headerBytes) {
73      this.encodingAlgo = encoding;
74      this.compressionAlgorithm =
75          compressionAlgorithm == null ? NONE : compressionAlgorithm;
76      if (this.compressionAlgorithm != NONE) {
77        compressor = compressionAlgorithm.getCompressor();
78        compressedByteStream = new ByteArrayOutputStream();
79        try {
80          compressionStream =
81              compressionAlgorithm.createPlainCompressionStream(
82                  compressedByteStream, compressor);
83        } catch (IOException e) {
84          throw new RuntimeException(
85              "Could not create compression stream for algorithm "
86                  + compressionAlgorithm, e);
87        }
88      }
89      dummyHeader = Preconditions.checkNotNull(headerBytes,
90        "Please pass HConstants.HFILEBLOCK_DUMMY_HEADER instead of null for param headerBytes");
91    }
92  
93    @Override
94    public void setDummyHeader(byte[] headerBytes) {
95      dummyHeader = headerBytes;
96    }
97  
98    /**
99     * prepare to start a new encoding.
100    * @throws IOException
101    */
102   public void prepareEncoding() throws IOException {
103     encodedStream.reset();
104     dataOut.write(dummyHeader);
105     if (encodingAlgo != null
106         && encodingAlgo != DataBlockEncoding.NONE) {
107       encodingAlgo.writeIdInBytes(dataOut);
108     }
109   }
110 
111   @Override
112   public void postEncoding(BlockType blockType)
113       throws IOException {
114     dataOut.flush();
115     compressAfterEncodingWithBlockType(encodedStream.toByteArray(), blockType);
116     this.blockType = blockType;
117   }
118 
119   /**
120    * @param uncompressedBytesWithHeader
121    * @param blockType
122    * @throws IOException
123    */
124   public void compressAfterEncodingWithBlockType(byte[] uncompressedBytesWithHeader,
125       BlockType blockType) throws IOException {
126     compressAfterEncoding(uncompressedBytesWithHeader, blockType, dummyHeader);
127   }
128 
129   /**
130    * @param uncompressedBytesWithHeader
131    * @param blockType
132    * @param headerBytes
133    * @throws IOException
134    */
135   protected void compressAfterEncoding(byte[] uncompressedBytesWithHeader,
136       BlockType blockType, byte[] headerBytes) throws IOException {
137     this.uncompressedBytesWithHeader = uncompressedBytesWithHeader;
138     if (compressionAlgorithm != NONE) {
139       compressedByteStream.reset();
140       compressedByteStream.write(headerBytes);
141       compressionStream.resetState();
142       compressionStream.write(uncompressedBytesWithHeader,
143           headerBytes.length, uncompressedBytesWithHeader.length
144               - headerBytes.length);
145 
146       compressionStream.flush();
147       compressionStream.finish();
148       onDiskBytesWithHeader = compressedByteStream.toByteArray();
149     } else {
150       onDiskBytesWithHeader = uncompressedBytesWithHeader;
151     }
152     this.blockType = blockType;
153   }
154 
155   @Override
156   public byte[] getOnDiskBytesWithHeader() {
157     return onDiskBytesWithHeader;
158   }
159 
160   @Override
161   public byte[] getUncompressedBytesWithHeader() {
162     return uncompressedBytesWithHeader;
163   }
164 
165   @Override
166   public BlockType getBlockType() {
167     return blockType;
168   }
169 
170   /**
171    * Releases the compressor this writer uses to compress blocks into the
172    * compressor pool.
173    */
174   @Override
175   public void close() {
176     if (compressor != null) {
177       compressionAlgorithm.returnCompressor(compressor);
178       compressor = null;
179     }
180   }
181 
182   @Override
183   public Algorithm getCompression() {
184     return this.compressionAlgorithm;
185   }
186 
187   public DataOutputStream getOutputStreamForEncoder() {
188     return this.dataOut;
189   }
190 
191   @Override
192   public DataBlockEncoding getDataBlockEncoding() {
193     return this.encodingAlgo;
194   }
195 }