1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.encoding;
18
19 import java.io.ByteArrayInputStream;
20 import java.io.ByteArrayOutputStream;
21 import java.io.DataInputStream;
22 import java.io.DataOutputStream;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25 import java.util.Iterator;
26
27 import org.apache.commons.lang.NotImplementedException;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.io.compress.Compressor;
30
31
32
33
34
35 public class EncodedDataBlock {
36 private static final int BUFFER_SIZE = 4 * 1024;
37 protected DataBlockEncoder dataBlockEncoder;
38 ByteArrayOutputStream uncompressedOutputStream;
39 ByteBuffer uncompressedBuffer;
40 private byte[] cacheCompressData;
41 private ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
42 private boolean includesMemstoreTS;
43
44
45
46
47
48 public EncodedDataBlock(DataBlockEncoder dataBlockEncoder,
49 boolean includesMemstoreTS) {
50 this.dataBlockEncoder = dataBlockEncoder;
51 uncompressedOutputStream = new ByteArrayOutputStream(BUFFER_SIZE);
52 }
53
54
55
56
57
58 public void addKv(KeyValue kv) {
59 cacheCompressData = null;
60 uncompressedOutputStream.write(
61 kv.getBuffer(), kv.getOffset(), kv.getLength());
62 }
63
64
65
66
67
68 public Iterator<KeyValue> getIterator() {
69 final int uncompressedSize = uncompressedOutputStream.size();
70 final ByteArrayInputStream bais = new ByteArrayInputStream(
71 getCompressedData());
72 final DataInputStream dis = new DataInputStream(bais);
73
74
75 return new Iterator<KeyValue>() {
76 private ByteBuffer decompressedData = null;
77
78 @Override
79 public boolean hasNext() {
80 if (decompressedData == null) {
81 return uncompressedSize > 0;
82 }
83 return decompressedData.hasRemaining();
84 }
85
86 @Override
87 public KeyValue next() {
88 if (decompressedData == null) {
89 try {
90 decompressedData = dataBlockEncoder.uncompressKeyValues(
91 dis, includesMemstoreTS);
92 } catch (IOException e) {
93 throw new RuntimeException("Problem with data block encoder, " +
94 "most likely it requested more bytes than are available.", e);
95 }
96 decompressedData.rewind();
97 }
98
99 int offset = decompressedData.position();
100 KeyValue kv = new KeyValue(decompressedData.array(), offset);
101 decompressedData.position(offset + kv.getLength());
102
103 return kv;
104 }
105
106 @Override
107 public void remove() {
108 throw new NotImplementedException("remove() is not supported!");
109 }
110
111 @Override
112 public String toString() {
113 return "Iterator of: " + dataBlockEncoder.getClass().getName();
114 }
115
116 };
117 }
118
119
120
121
122
123 public int getSize() {
124 return getCompressedData().length;
125 }
126
127
128
129
130
131
132
133
134
135
136 public static int checkCompressedSize(Compressor compressor, byte[] buffer,
137 int offset, int length) {
138 byte[] compressedBuffer = new byte[buffer.length];
139
140 compressor.setInput(buffer, offset, length);
141 compressor.finish();
142 int currentPos = 0;
143 while (!compressor.finished()) {
144 try {
145
146
147 currentPos += compressor.compress(compressedBuffer, 0,
148 compressedBuffer.length);
149 } catch (IOException e) {
150 throw new RuntimeException(
151 "For some reason compressor couldn't read data. " +
152 "It is likely a problem with " +
153 compressor.getClass().getName(), e);
154 }
155 }
156 return currentPos;
157 }
158
159
160
161
162
163
164 public int checkCompressedSize(Compressor compressor) {
165
166 byte[] compressedBytes = getCompressedData();
167 return checkCompressedSize(compressor, compressedBytes, 0,
168 compressedBytes.length);
169 }
170
171 private byte[] getCompressedData() {
172
173 if (cacheCompressData != null) {
174 return cacheCompressData;
175 }
176 cacheCompressData = doCompressData();
177
178 return cacheCompressData;
179 }
180
181 private ByteBuffer getUncompressedBuffer() {
182 if (uncompressedBuffer == null ||
183 uncompressedBuffer.limit() < uncompressedOutputStream.size()) {
184 uncompressedBuffer = ByteBuffer.wrap(
185 uncompressedOutputStream.toByteArray());
186 }
187 return uncompressedBuffer;
188 }
189
190
191
192
193
194 public byte[] doCompressData() {
195 compressedStream.reset();
196 DataOutputStream dataOut = new DataOutputStream(compressedStream);
197 try {
198 this.dataBlockEncoder.compressKeyValues(
199 dataOut, getUncompressedBuffer(), includesMemstoreTS);
200 } catch (IOException e) {
201 throw new RuntimeException(String.format(
202 "Bug in decoding part of algorithm %s. " +
203 "Probably it requested more bytes than are available.",
204 toString()), e);
205 }
206 return compressedStream.toByteArray();
207 }
208
209 @Override
210 public String toString() {
211 return dataBlockEncoder.toString();
212 }
213
214
215
216
217
218 public byte[] getRawKeyValues() {
219 return uncompressedOutputStream.toByteArray();
220 }
221 }