1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.encoding;
18
19 import java.io.ByteArrayInputStream;
20 import java.io.DataInputStream;
21 import java.io.DataOutputStream;
22 import java.io.IOException;
23 import java.io.OutputStream;
24 import java.nio.ByteBuffer;
25 import java.util.Iterator;
26
27 import org.apache.commons.lang.NotImplementedException;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.hbase.Cell;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.KeyValue;
32 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
33 import org.apache.hadoop.hbase.io.hfile.HFileContext;
34 import org.apache.hadoop.hbase.util.ByteBufferUtils;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.apache.hadoop.io.IOUtils;
37 import org.apache.hadoop.io.compress.Compressor;
38
39 import com.google.common.base.Preconditions;
40
41
42
43
44
45 @InterfaceAudience.Private
46 public class EncodedDataBlock {
47 private byte[] rawKVs;
48 private ByteBuffer rawBuffer;
49 private DataBlockEncoder dataBlockEncoder;
50
51 private byte[] cachedEncodedData;
52
53 private final HFileBlockEncodingContext encodingCtx;
54 private HFileContext meta;
55
56
57
58
59
60
61
62
63 public EncodedDataBlock(DataBlockEncoder dataBlockEncoder, DataBlockEncoding encoding,
64 byte[] rawKVs, HFileContext meta) {
65 Preconditions.checkNotNull(encoding,
66 "Cannot create encoded data block with null encoder");
67 this.dataBlockEncoder = dataBlockEncoder;
68 encodingCtx = dataBlockEncoder.newDataBlockEncodingContext(encoding,
69 HConstants.HFILEBLOCK_DUMMY_HEADER, meta);
70 this.rawKVs = rawKVs;
71 this.meta = meta;
72 }
73
74
75
76
77
78
79 public Iterator<Cell> getIterator(int headerSize) {
80 final int rawSize = rawKVs.length;
81 byte[] encodedDataWithHeader = getEncodedData();
82 int bytesToSkip = headerSize + Bytes.SIZEOF_SHORT;
83 ByteArrayInputStream bais = new ByteArrayInputStream(encodedDataWithHeader,
84 bytesToSkip, encodedDataWithHeader.length - bytesToSkip);
85 final DataInputStream dis = new DataInputStream(bais);
86
87 return new Iterator<Cell>() {
88 private ByteBuffer decompressedData = null;
89
90 @Override
91 public boolean hasNext() {
92 if (decompressedData == null) {
93 return rawSize > 0;
94 }
95 return decompressedData.hasRemaining();
96 }
97
98 @Override
99 public Cell next() {
100 if (decompressedData == null) {
101 try {
102 decompressedData = dataBlockEncoder.decodeKeyValues(dis, dataBlockEncoder
103 .newDataBlockDecodingContext(meta));
104 } catch (IOException e) {
105 throw new RuntimeException("Problem with data block encoder, " +
106 "most likely it requested more bytes than are available.", e);
107 }
108 decompressedData.rewind();
109 }
110 int offset = decompressedData.position();
111 int klen = decompressedData.getInt();
112 int vlen = decompressedData.getInt();
113 int tagsLen = 0;
114 ByteBufferUtils.skip(decompressedData, klen + vlen);
115
116 if (meta.isIncludesTags()) {
117 tagsLen = ((decompressedData.get() & 0xff) << 8) ^ (decompressedData.get() & 0xff);
118 ByteBufferUtils.skip(decompressedData, tagsLen);
119 }
120 KeyValue kv = new KeyValue(decompressedData.array(), offset,
121 (int) KeyValue.getKeyValueDataStructureSize(klen, vlen, tagsLen));
122 if (meta.isIncludesMvcc()) {
123 long mvccVersion = ByteBufferUtils.readVLong(decompressedData);
124 kv.setMvccVersion(mvccVersion);
125 }
126 return kv;
127 }
128
129 @Override
130 public void remove() {
131 throw new NotImplementedException("remove() is not supported!");
132 }
133
134 @Override
135 public String toString() {
136 return "Iterator of: " + dataBlockEncoder.getClass().getName();
137 }
138
139 };
140 }
141
142
143
144
145
146 public int getSize() {
147 return getEncodedData().length;
148 }
149
150
151
152
153
154
155
156
157
158
159
160
161 public static int getCompressedSize(Algorithm algo, Compressor compressor,
162 byte[] inputBuffer, int offset, int length) throws IOException {
163
164
165
166 final IOUtils.NullOutputStream nullOutputStream = new IOUtils.NullOutputStream();
167 final DataOutputStream compressedStream = new DataOutputStream(nullOutputStream);
168 OutputStream compressingStream = null;
169
170
171 try {
172 if (compressor != null) {
173 compressor.reset();
174 }
175
176 compressingStream = algo.createCompressionStream(compressedStream, compressor, 0);
177
178 compressingStream.write(inputBuffer, offset, length);
179 compressingStream.flush();
180
181 return compressedStream.size();
182 } finally {
183 nullOutputStream.close();
184 compressedStream.close();
185 if (compressingStream != null) compressingStream.close();
186 }
187 }
188
189
190
191
192
193
194
195
196 public int getEncodedCompressedSize(Algorithm comprAlgo,
197 Compressor compressor) throws IOException {
198 byte[] compressedBytes = getEncodedData();
199 return getCompressedSize(comprAlgo, compressor, compressedBytes, 0,
200 compressedBytes.length);
201 }
202
203
204 private byte[] getEncodedData() {
205 if (cachedEncodedData != null) {
206 return cachedEncodedData;
207 }
208 cachedEncodedData = encodeData();
209 return cachedEncodedData;
210 }
211
212 private ByteBuffer getUncompressedBuffer() {
213 if (rawBuffer == null || rawBuffer.limit() < rawKVs.length) {
214 rawBuffer = ByteBuffer.wrap(rawKVs);
215 }
216 return rawBuffer;
217 }
218
219
220
221
222
223 public byte[] encodeData() {
224 try {
225 this.dataBlockEncoder.encodeKeyValues(
226 getUncompressedBuffer(), encodingCtx);
227 } catch (IOException e) {
228 throw new RuntimeException(String.format(
229 "Bug in encoding part of algorithm %s. " +
230 "Probably it requested more bytes than are available.",
231 toString()), e);
232 }
233 return encodingCtx.getUncompressedBytesWithHeader();
234 }
235
236 @Override
237 public String toString() {
238 return dataBlockEncoder.toString();
239 }
240 }