View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver.wal;
19  
20  import java.io.ByteArrayInputStream;
21  import java.io.ByteArrayOutputStream;
22  import java.io.EOFException;
23  import java.io.IOException;
24  import java.io.InputStream;
25  import java.io.OutputStream;
26  import java.security.SecureRandom;
27  
28  import org.apache.commons.io.IOUtils;
29  import org.apache.hadoop.hbase.classification.InterfaceAudience;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.KeyValueUtil;
34  import org.apache.hadoop.hbase.codec.KeyValueCodec;
35  import org.apache.hadoop.hbase.io.crypto.Decryptor;
36  import org.apache.hadoop.hbase.io.crypto.Encryption;
37  import org.apache.hadoop.hbase.io.crypto.Encryptor;
38  import org.apache.hadoop.hbase.io.util.StreamUtils;
39  import org.apache.hadoop.hbase.util.Bytes;
40  
41  /**
42   * A WALCellCodec that encrypts the WALedits.
43   */
44  @InterfaceAudience.Private
45  public class SecureWALCellCodec extends WALCellCodec {
46  
47    private Encryptor encryptor;
48    private Decryptor decryptor;
49  
50    public SecureWALCellCodec(Configuration conf, CompressionContext compression) {
51      super(conf, compression);
52    }
53  
54    public SecureWALCellCodec(Configuration conf, Encryptor encryptor) {
55      super(conf, null);
56      this.encryptor = encryptor;
57    }
58  
59    public SecureWALCellCodec(Configuration conf, Decryptor decryptor) {
60      super(conf, null);
61      this.decryptor = decryptor;
62    }
63  
64    static class EncryptedKvDecoder extends KeyValueCodec.KeyValueDecoder {
65  
66      private Decryptor decryptor;
67      private byte[] iv;
68  
69      public EncryptedKvDecoder(InputStream in) {
70        super(in);
71      }
72  
73      public EncryptedKvDecoder(InputStream in, Decryptor decryptor) {
74        super(in);
75        this.decryptor = decryptor;
76        if (decryptor != null) {
77          this.iv = new byte[decryptor.getIvLength()];
78        }
79      }
80  
81      @Override
82      protected Cell parseCell() throws IOException {
83        if (this.decryptor == null) {
84          return super.parseCell();
85        }
86        int ivLength = 0;
87        try {
88          ivLength = StreamUtils.readRawVarint32(in);
89        } catch (EOFException e) {
90          // EOF at start is OK
91          return null;
92        }
93  
94        // TODO: An IV length of 0 could signify an unwrapped cell, when the
95        // encoder supports that just read the remainder in directly
96  
97        if (ivLength != this.iv.length) {
98          throw new IOException("Incorrect IV length: expected=" + iv.length + " have=" +
99            ivLength);
100       }
101       IOUtils.readFully(in, this.iv);
102 
103       int codedLength = StreamUtils.readRawVarint32(in);
104       byte[] codedBytes = new byte[codedLength];
105       IOUtils.readFully(in, codedBytes);
106 
107       decryptor.setIv(iv);
108       decryptor.reset();
109 
110       InputStream cin = decryptor.createDecryptionStream(new ByteArrayInputStream(codedBytes));
111 
112       // TODO: Add support for WAL compression
113 
114       int keylength = StreamUtils.readRawVarint32(cin);
115       int vlength = StreamUtils.readRawVarint32(cin);
116       int tagsLength = StreamUtils.readRawVarint32(cin);
117       int length = 0;
118       if (tagsLength == 0) {
119         length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength;
120       } else {
121         length = KeyValue.KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE + keylength + vlength + tagsLength;
122       }
123 
124       byte[] backingArray = new byte[length];
125       int pos = 0;
126       pos = Bytes.putInt(backingArray, pos, keylength);
127       pos = Bytes.putInt(backingArray, pos, vlength);
128 
129       // Row
130       int elemLen = StreamUtils.readRawVarint32(cin);
131       pos = Bytes.putShort(backingArray, pos, (short)elemLen);
132       IOUtils.readFully(cin, backingArray, pos, elemLen);
133       pos += elemLen;
134       // Family
135       elemLen = StreamUtils.readRawVarint32(cin);
136       pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
137       IOUtils.readFully(cin, backingArray, pos, elemLen);
138       pos += elemLen;
139       // Qualifier
140       elemLen = StreamUtils.readRawVarint32(cin);
141       IOUtils.readFully(cin, backingArray, pos, elemLen);
142       pos += elemLen;
143       // Remainder
144       IOUtils.readFully(cin, backingArray, pos, length - pos);
145       return new KeyValue(backingArray, 0, length);
146     }
147 
148   }
149 
150   static class EncryptedKvEncoder extends KeyValueCodec.KeyValueEncoder {
151 
152     private Encryptor encryptor;
153     private final ThreadLocal<byte[]> iv = new ThreadLocal<byte[]>() {
154       @Override
155       protected byte[] initialValue() {
156         byte[] iv = new byte[encryptor.getIvLength()];
157         new SecureRandom().nextBytes(iv);
158         return iv;
159       }
160     };
161 
162     protected byte[] nextIv() {
163       byte[] b = iv.get(), ret = new byte[b.length];
164       System.arraycopy(b, 0, ret, 0, b.length);
165       return ret;
166     }
167 
168     protected void incrementIv(int v) {
169       Encryption.incrementIv(iv.get(), 1 + (v / encryptor.getBlockSize()));
170     }
171 
172     public EncryptedKvEncoder(OutputStream os) {
173       super(os);
174     }
175 
176     public EncryptedKvEncoder(OutputStream os, Encryptor encryptor) {
177       super(os);
178       this.encryptor = encryptor;
179     }
180 
181     @Override
182     public void write(Cell cell) throws IOException {
183       if (encryptor == null) {
184         super.write(cell);
185         return;
186       }
187 
188       byte[] iv = nextIv();
189       encryptor.setIv(iv);
190       encryptor.reset();
191 
192       // TODO: Check if this is a cell for an encrypted CF. If not, we can
193       // write a 0 here to signal an unwrapped cell and just dump the KV bytes
194       // afterward
195 
196       StreamUtils.writeRawVInt32(out, iv.length);
197       out.write(iv);
198 
199       // TODO: Add support for WAL compression
200 
201       ByteArrayOutputStream baos = new ByteArrayOutputStream();
202       OutputStream cout = encryptor.createEncryptionStream(baos);
203 
204       int tlen = cell.getTagsLength();
205       // Write the KeyValue infrastructure as VInts.
206       StreamUtils.writeRawVInt32(cout, KeyValueUtil.keyLength(cell));
207       StreamUtils.writeRawVInt32(cout, cell.getValueLength());
208       // To support tags
209       StreamUtils.writeRawVInt32(cout, tlen);
210 
211       // Write row, qualifier, and family
212       StreamUtils.writeRawVInt32(cout, cell.getRowLength());
213       cout.write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
214       StreamUtils.writeRawVInt32(cout, cell.getFamilyLength());
215       cout.write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
216       StreamUtils.writeRawVInt32(cout, cell.getQualifierLength());
217       cout.write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
218       // Write the rest ie. ts, type, value and tags parts
219       StreamUtils.writeLong(cout, cell.getTimestamp());
220       cout.write(cell.getTypeByte());
221       cout.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
222       if (tlen > 0) {
223         cout.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
224       }
225       cout.close();
226 
227       StreamUtils.writeRawVInt32(out, baos.size());
228       baos.writeTo(out);
229 
230       // Increment IV given the final payload length
231       incrementIv(baos.size());
232     }
233 
234   }
235 
236   @Override
237   public Decoder getDecoder(InputStream is) {
238     return new EncryptedKvDecoder(is, decryptor);
239   }
240 
241   @Override
242   public Encoder getEncoder(OutputStream os) {
243     return new EncryptedKvEncoder(os, encryptor);
244   }
245 
246   public static WALCellCodec getCodec(Configuration conf, Encryptor encryptor) {
247     return new SecureWALCellCodec(conf, encryptor);
248   }
249 
250   public static WALCellCodec getCodec(Configuration conf, Decryptor decryptor) {
251     return new SecureWALCellCodec(conf, decryptor);
252   }
253 
254 }