1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.wal;
19
20 import java.io.ByteArrayInputStream;
21 import java.io.ByteArrayOutputStream;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.OutputStream;
25 import java.security.SecureRandom;
26
27 import org.apache.commons.io.IOUtils;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.KeyValue;
32 import org.apache.hadoop.hbase.codec.KeyValueCodec;
33 import org.apache.hadoop.hbase.io.crypto.Decryptor;
34 import org.apache.hadoop.hbase.io.crypto.Encryption;
35 import org.apache.hadoop.hbase.io.crypto.Encryptor;
36 import org.apache.hadoop.hbase.io.util.StreamUtils;
37 import org.apache.hadoop.hbase.util.Bytes;
38
39
40
41
42 @InterfaceAudience.Private
43 public class SecureWALCellCodec extends WALCellCodec {
44
45 private Encryptor encryptor;
46 private Decryptor decryptor;
47
48 public SecureWALCellCodec(Configuration conf, CompressionContext compression) {
49 super(conf, compression);
50 }
51
52 public SecureWALCellCodec(Configuration conf, Encryptor encryptor) {
53 super(conf, null);
54 this.encryptor = encryptor;
55 }
56
57 public SecureWALCellCodec(Configuration conf, Decryptor decryptor) {
58 super(conf, null);
59 this.decryptor = decryptor;
60 }
61
62 static class EncryptedKvDecoder extends KeyValueCodec.KeyValueDecoder {
63
64 private Decryptor decryptor;
65 private byte[] iv;
66
67 public EncryptedKvDecoder(InputStream in) {
68 super(in);
69 }
70
71 public EncryptedKvDecoder(InputStream in, Decryptor decryptor) {
72 super(in);
73 this.decryptor = decryptor;
74 if (decryptor != null) {
75 this.iv = new byte[decryptor.getIvLength()];
76 }
77 }
78
79 @Override
80 protected Cell parseCell() throws IOException {
81 if (this.decryptor == null) {
82 return super.parseCell();
83 }
84 int ivLength = 0;
85
86 ivLength = StreamUtils.readRawVarint32(in);
87
88
89
90
91 if (ivLength != this.iv.length) {
92 throw new IOException("Incorrect IV length: expected=" + iv.length + " have=" +
93 ivLength);
94 }
95 IOUtils.readFully(in, this.iv);
96
97 int codedLength = StreamUtils.readRawVarint32(in);
98 byte[] codedBytes = new byte[codedLength];
99 IOUtils.readFully(in, codedBytes);
100
101 decryptor.setIv(iv);
102 decryptor.reset();
103
104 InputStream cin = decryptor.createDecryptionStream(new ByteArrayInputStream(codedBytes));
105
106
107
108 int keylength = StreamUtils.readRawVarint32(cin);
109 int vlength = StreamUtils.readRawVarint32(cin);
110 int tagsLength = StreamUtils.readRawVarint32(cin);
111 int length = 0;
112 if (tagsLength == 0) {
113 length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
114 } else {
115 length = KeyValue.KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE + keylength + vlength + tagsLength;
116 }
117
118 byte[] backingArray = new byte[length];
119 int pos = 0;
120 pos = Bytes.putInt(backingArray, pos, keylength);
121 pos = Bytes.putInt(backingArray, pos, vlength);
122
123
124 int elemLen = StreamUtils.readRawVarint32(cin);
125 pos = Bytes.putShort(backingArray, pos, (short)elemLen);
126 IOUtils.readFully(cin, backingArray, pos, elemLen);
127 pos += elemLen;
128
129 elemLen = StreamUtils.readRawVarint32(cin);
130 pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
131 IOUtils.readFully(cin, backingArray, pos, elemLen);
132 pos += elemLen;
133
134 elemLen = StreamUtils.readRawVarint32(cin);
135 IOUtils.readFully(cin, backingArray, pos, elemLen);
136 pos += elemLen;
137
138 IOUtils.readFully(cin, backingArray, pos, length - pos);
139 return new KeyValue(backingArray, 0, length);
140 }
141
142 }
143
144 static class EncryptedKvEncoder extends KeyValueCodec.KeyValueEncoder {
145
146 private Encryptor encryptor;
147 private final ThreadLocal<byte[]> iv = new ThreadLocal<byte[]>() {
148 @Override
149 protected byte[] initialValue() {
150 byte[] iv = new byte[encryptor.getIvLength()];
151 new SecureRandom().nextBytes(iv);
152 return iv;
153 }
154 };
155
156 protected byte[] nextIv() {
157 byte[] b = iv.get(), ret = new byte[b.length];
158 System.arraycopy(b, 0, ret, 0, b.length);
159 return ret;
160 }
161
162 protected void incrementIv(int v) {
163 Encryption.incrementIv(iv.get(), 1 + (v / encryptor.getBlockSize()));
164 }
165
166 public EncryptedKvEncoder(OutputStream os) {
167 super(os);
168 }
169
170 public EncryptedKvEncoder(OutputStream os, Encryptor encryptor) {
171 super(os);
172 this.encryptor = encryptor;
173 }
174
175 @Override
176 public void write(Cell cell) throws IOException {
177 if (!(cell instanceof KeyValue)) throw new IOException("Cannot write non-KV cells to WAL");
178 if (encryptor == null) {
179 super.write(cell);
180 return;
181 }
182
183 KeyValue kv = (KeyValue)cell;
184 byte[] kvBuffer = kv.getBuffer();
185 int offset = kv.getOffset();
186
187 byte[] iv = nextIv();
188 encryptor.setIv(iv);
189 encryptor.reset();
190
191
192
193
194
195 StreamUtils.writeRawVInt32(out, iv.length);
196 out.write(iv);
197
198
199
200 ByteArrayOutputStream baos = new ByteArrayOutputStream();
201 OutputStream cout = encryptor.createEncryptionStream(baos);
202
203
204 StreamUtils.writeRawVInt32(cout, kv.getKeyLength());
205 StreamUtils.writeRawVInt32(cout, kv.getValueLength());
206
207 StreamUtils.writeRawVInt32(cout, kv.getTagsLengthUnsigned());
208
209
210 StreamUtils.writeRawVInt32(cout, kv.getRowLength());
211 cout.write(kvBuffer, kv.getRowOffset(), kv.getRowLength());
212 StreamUtils.writeRawVInt32(cout, kv.getFamilyLength());
213 cout.write(kvBuffer, kv.getFamilyOffset(), kv.getFamilyLength());
214 StreamUtils.writeRawVInt32(cout, kv.getQualifierLength());
215 cout.write(kvBuffer, kv.getQualifierOffset(), kv.getQualifierLength());
216
217 int pos = kv.getTimestampOffset();
218 int remainingLength = kv.getLength() + offset - pos;
219 cout.write(kvBuffer, pos, remainingLength);
220 cout.close();
221
222 StreamUtils.writeRawVInt32(out, baos.size());
223 baos.writeTo(out);
224
225
226 incrementIv(baos.size());
227 }
228
229 }
230
231 @Override
232 public Decoder getDecoder(InputStream is) {
233 return new EncryptedKvDecoder(is, decryptor);
234 }
235
236 @Override
237 public Encoder getEncoder(OutputStream os) {
238 return new EncryptedKvEncoder(os, encryptor);
239 }
240
241 public static WALCellCodec getCodec(Configuration conf, Encryptor encryptor) {
242 return new SecureWALCellCodec(conf, encryptor);
243 }
244
245 public static WALCellCodec getCodec(Configuration conf, Decryptor decryptor) {
246 return new SecureWALCellCodec(conf, decryptor);
247 }
248
249 }