View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations under
15   * the License.
16   */
17  package org.apache.hadoop.hbase.io.crypto;
18  
19  import java.io.IOException;
20  import java.io.InputStream;
21  import java.io.OutputStream;
22  import java.security.DigestException;
23  import java.security.Key;
24  import java.security.MessageDigest;
25  import java.security.NoSuchAlgorithmException;
26  import java.security.spec.InvalidKeySpecException;
27  import java.util.Map;
28  import java.util.concurrent.ConcurrentHashMap;
29  
30  import javax.crypto.SecretKeyFactory;
31  import javax.crypto.spec.PBEKeySpec;
32  import javax.crypto.spec.SecretKeySpec;
33  
34  import org.apache.commons.io.IOUtils;
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.hbase.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.Pair;
44  import org.apache.hadoop.util.ReflectionUtils;
45  
46  /**
47   * A facade for encryption algorithms and related support.
48   */
49  @InterfaceAudience.Public
50  @InterfaceStability.Unstable
51  public final class Encryption {
52  
53    private static final Log LOG = LogFactory.getLog(Encryption.class);
54  
55    /**
56     * Crypto context
57     */
58    @InterfaceAudience.Public
59    @InterfaceStability.Unstable
60    public static class Context extends org.apache.hadoop.hbase.io.crypto.Context {
61  
62      /** The null crypto context */
63      public static final Context NONE = new Context();
64  
65      private Context() {
66        super();
67      }
68  
69      private Context(Configuration conf) {
70        super(conf);
71      }
72  
73      @Override
74      public Context setCipher(Cipher cipher) {
75        super.setCipher(cipher);
76        return this;
77      }
78  
79      @Override
80      public Context setKey(Key key) {
81        super.setKey(key);
82        return this;
83      }
84  
85      public Context setKey(byte[] key) {
86        super.setKey(new SecretKeySpec(key, getCipher().getName()));
87        return this;
88      }
89    }
90  
91    public static Context newContext() {
92      return new Context();
93    }
94  
95    public static Context newContext(Configuration conf) {
96      return new Context(conf);
97    }
98  
99    // Prevent instantiation
100   private Encryption() {
101     super();
102   }
103 
104   /**
105    * Get an cipher given a name
106    * @param name the cipher name
107    * @return the cipher, or null if a suitable one could not be found
108    */
109   public static Cipher getCipher(Configuration conf, String name) {
110     return getCipherProvider(conf).getCipher(name);
111   }
112 
113   /**
114    * Get names of supported encryption algorithms
115    *
116    * @return Array of strings, each represents a supported encryption algorithm
117    */
118   public static String[] getSupportedCiphers() {
119     return getSupportedCiphers(HBaseConfiguration.create());
120   }
121 
122   /**
123    * Get names of supported encryption algorithms
124    *
125    * @return Array of strings, each represents a supported encryption algorithm
126    */
127   public static String[] getSupportedCiphers(Configuration conf) {
128     return getCipherProvider(conf).getSupportedCiphers();
129   }
130 
131   /**
132    * Return the MD5 digest of the concatenation of the supplied arguments.
133    */
134   public static byte[] hash128(String... args) {
135     byte[] result = new byte[16];
136     try {
137       MessageDigest md = MessageDigest.getInstance("MD5");
138       for (String arg: args) {
139         md.update(Bytes.toBytes(arg));
140       }
141       md.digest(result, 0, result.length);
142       return result;
143     } catch (NoSuchAlgorithmException e) {
144       throw new RuntimeException(e);
145     } catch (DigestException e) {
146       throw new RuntimeException(e);
147     }
148   }
149 
150   /**
151    * Return the MD5 digest of the concatenation of the supplied arguments.
152    */
153   public static byte[] hash128(byte[]... args) {
154     byte[] result = new byte[16];
155     try {
156       MessageDigest md = MessageDigest.getInstance("MD5");
157       for (byte[] arg: args) {
158         md.update(arg);
159       }
160       md.digest(result, 0, result.length);
161       return result;
162     } catch (NoSuchAlgorithmException e) {
163       throw new RuntimeException(e);
164     } catch (DigestException e) {
165       throw new RuntimeException(e);
166     }
167   }
168 
169   /**
170    * Return the SHA-256 digest of the concatenation of the supplied arguments.
171    */
172   public static byte[] hash256(String... args) {
173     byte[] result = new byte[32];
174     try {
175       MessageDigest md = MessageDigest.getInstance("SHA-256");
176       for (String arg: args) {
177         md.update(Bytes.toBytes(arg));
178       }
179       md.digest(result, 0, result.length);
180       return result;
181     } catch (NoSuchAlgorithmException e) {
182       throw new RuntimeException(e);
183     } catch (DigestException e) {
184       throw new RuntimeException(e);
185     }
186   }
187 
188   /**
189    * Return the SHA-256 digest of the concatenation of the supplied arguments.
190    */
191   public static byte[] hash256(byte[]... args) {
192     byte[] result = new byte[32];
193     try {
194       MessageDigest md = MessageDigest.getInstance("SHA-256");
195       for (byte[] arg: args) {
196         md.update(arg);
197       }
198       md.digest(result, 0, result.length);
199       return result;
200     } catch (NoSuchAlgorithmException e) {
201       throw new RuntimeException(e);
202     } catch (DigestException e) {
203       throw new RuntimeException(e);
204     }
205   }
206 
207   /**
208    * Return a 128 bit key derived from the concatenation of the supplied
209    * arguments using PBKDF2WithHmacSHA1 at 10,000 iterations.
210    * 
211    */
212   public static byte[] pbkdf128(String... args) {
213     byte[] salt = new byte[128];
214     Bytes.random(salt);
215     StringBuilder sb = new StringBuilder();
216     for (String s: args) {
217       sb.append(s);
218     }
219     PBEKeySpec spec = new PBEKeySpec(sb.toString().toCharArray(), salt, 10000, 128);
220     try {
221       return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1")
222         .generateSecret(spec).getEncoded();
223     } catch (NoSuchAlgorithmException e) {
224       throw new RuntimeException(e);
225     } catch (InvalidKeySpecException e) {
226       throw new RuntimeException(e);
227     }
228   }
229 
230   /**
231    * Return a 128 bit key derived from the concatenation of the supplied
232    * arguments using PBKDF2WithHmacSHA1 at 10,000 iterations.
233    * 
234    */
235   public static byte[] pbkdf128(byte[]... args) {
236     byte[] salt = new byte[128];
237     Bytes.random(salt);
238     StringBuilder sb = new StringBuilder();
239     for (byte[] b: args) {
240       sb.append(b);
241     }
242     PBEKeySpec spec = new PBEKeySpec(sb.toString().toCharArray(), salt, 10000, 128);
243     try {
244       return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1")
245         .generateSecret(spec).getEncoded();
246     } catch (NoSuchAlgorithmException e) {
247       throw new RuntimeException(e);
248     } catch (InvalidKeySpecException e) {
249       throw new RuntimeException(e);
250     }
251   }
252 
253   /**
254    * Encrypt a block of plaintext
255    * <p>
256    * The encryptor's state will be finalized. It should be reinitialized or
257    * returned to the pool.
258    * @param out ciphertext
259    * @param src plaintext
260    * @param offset
261    * @param length
262    * @param e
263    * @throws IOException
264     */
265   public static void encrypt(OutputStream out, byte[] src, int offset,
266       int length, Encryptor e) throws IOException {
267     OutputStream cout = e.createEncryptionStream(out);
268     try {
269       cout.write(src, offset, length);
270     } finally {
271       cout.close();
272     }
273   }
274 
275   /**
276    * Encrypt a block of plaintext
277    * @param out ciphertext
278    * @param src plaintext
279    * @param offset
280    * @param length
281    * @param context
282    * @param iv
283    * @throws IOException
284     */
285   public static void encrypt(OutputStream out, byte[] src, int offset,
286       int length, Context context, byte[] iv) throws IOException {
287     Encryptor e = context.getCipher().getEncryptor();
288     e.setKey(context.getKey());
289     e.setIv(iv); // can be null
290     e.reset();
291     encrypt(out, src, offset, length, e);
292   }
293 
294   /**
295    * Encrypt a stream of plaintext given an encryptor
296    * <p>
297    * The encryptor's state will be finalized. It should be reinitialized or
298    * returned to the pool.
299    * @param out ciphertext
300    * @param in plaintext
301    * @param e
302    * @throws IOException
303    */
304   public static void encrypt(OutputStream out, InputStream in, Encryptor e)
305       throws IOException {
306     OutputStream cout = e.createEncryptionStream(out);
307     try {
308       IOUtils.copy(in, cout);
309     } finally {
310       cout.close();
311     }
312   }
313 
314   /**
315    * Encrypt a stream of plaintext given a context and IV
316    * @param out ciphertext
317    * @param in plaintet
318    * @param context
319    * @param iv
320    * @throws IOException
321    */
322   public static void encrypt(OutputStream out, InputStream in, Context context,
323       byte[] iv) throws IOException {
324     Encryptor e = context.getCipher().getEncryptor();
325     e.setKey(context.getKey());
326     e.setIv(iv); // can be null
327     e.reset();
328     encrypt(out, in, e);
329   }
330 
331   /**
332    * Decrypt a block of ciphertext read in from a stream with the given
333    * cipher and context
334    * <p>
335    * The decryptor's state will be finalized. It should be reinitialized or
336    * returned to the pool.
337    * @param dest
338    * @param destOffset
339    * @param in
340    * @param destSize
341    * @param d
342    * @throws IOException
343    */
344   public static void decrypt(byte[] dest, int destOffset, InputStream in,
345       int destSize, Decryptor d) throws IOException {
346     InputStream cin = d.createDecryptionStream(in);
347     try {
348       IOUtils.readFully(cin, dest, destOffset, destSize);
349     } finally {
350       cin.close();
351     }
352   }
353 
354   /**
355    * Decrypt a block of ciphertext from a stream given a context and IV
356    * @param dest
357    * @param destOffset
358    * @param in
359    * @param destSize
360    * @param context
361    * @param iv
362    * @throws IOException
363    */
364   public static void decrypt(byte[] dest, int destOffset, InputStream in,
365       int destSize, Context context, byte[] iv) throws IOException {
366     Decryptor d = context.getCipher().getDecryptor();
367     d.setKey(context.getKey());
368     d.setIv(iv); // can be null
369     decrypt(dest, destOffset, in, destSize, d);
370   }
371 
372   /**
373    * Decrypt a stream of ciphertext given a decryptor
374    * @param out
375    * @param in
376    * @param outLen
377    * @param d
378    * @throws IOException
379    */
380   public static void decrypt(OutputStream out, InputStream in, int outLen,
381       Decryptor d) throws IOException {
382     InputStream cin = d.createDecryptionStream(in);
383     byte buf[] = new byte[8*1024];
384     long remaining = outLen;
385     try {
386       while (remaining > 0) {
387         int toRead = (int)(remaining < buf.length ? remaining : buf.length);
388         int read = cin.read(buf, 0, toRead);
389         if (read < 0) {
390           break;
391         }
392         out.write(buf, 0, read);
393         remaining -= read;
394       }
395     } finally {
396       cin.close();
397     }
398   }
399 
400   /**
401    * Decrypt a stream of ciphertext given a context and IV
402    * @param out
403    * @param in
404    * @param outLen
405    * @param context
406    * @param iv
407    * @throws IOException
408    */
409   public static void decrypt(OutputStream out, InputStream in, int outLen,
410       Context context, byte[] iv) throws IOException {
411     Decryptor d = context.getCipher().getDecryptor();
412     d.setKey(context.getKey());
413     d.setIv(iv); // can be null
414     decrypt(out, in, outLen, d);
415   }
416 
417   /**
418    * Resolves a key for the given subject
419    * @param subject
420    * @param conf
421    * @return a key for the given subject
422    * @throws IOException if the key is not found
423    */
424   public static Key getSecretKeyForSubject(String subject, Configuration conf)
425       throws IOException {
426     KeyProvider provider = (KeyProvider)getKeyProvider(conf);
427     if (provider != null) try {
428       Key[] keys = provider.getKeys(new String[] { subject });
429       if (keys != null && keys.length > 0) {
430         return keys[0];
431       }
432     } catch (Exception e) {
433       throw new IOException(e);
434     }
435     throw new IOException("No key found for subject '" + subject + "'");
436   }
437 
438   /**
439    * Encrypts a block of plaintext with the symmetric key resolved for the given subject
440    * @param out ciphertext
441    * @param in plaintext
442    * @param conf configuration
443    * @param cipher the encryption algorithm
444    * @param iv the initialization vector, can be null
445    * @throws IOException
446    */
447   public static void encryptWithSubjectKey(OutputStream out, InputStream in,
448       String subject, Configuration conf, Cipher cipher, byte[] iv)
449       throws IOException {
450     Key key = getSecretKeyForSubject(subject, conf);
451     if (key == null) {
452       throw new IOException("No key found for subject '" + subject + "'");
453     }
454     Encryptor e = cipher.getEncryptor();
455     e.setKey(key);
456     e.setIv(iv); // can be null
457     encrypt(out, in, e);
458   }
459 
460   /**
461    * Decrypts a block of ciphertext with the symmetric key resolved for the given subject
462    * @param out plaintext
463    * @param in ciphertext
464    * @param outLen the expected plaintext length
465    * @param subject the subject's key alias
466    * @param conf configuration
467    * @param cipher the encryption algorithm
468    * @param iv the initialization vector, can be null
469    * @throws IOException
470    */
471   public static void decryptWithSubjectKey(OutputStream out, InputStream in, int outLen,
472       String subject, Configuration conf, Cipher cipher, byte[] iv) throws IOException {
473     Key key = getSecretKeyForSubject(subject, conf);
474     if (key == null) {
475       throw new IOException("No key found for subject '" + subject + "'");
476     }
477     Decryptor d = cipher.getDecryptor();
478     d.setKey(key);
479     d.setIv(iv); // can be null
480     try {
481       decrypt(out, in, outLen, d);
482     } catch (IOException e) {
483       // If the current cipher algorithm fails to unwrap, try the alternate cipher algorithm, if one
484       // is configured
485       String alternateAlgorithm = conf.get(HConstants.CRYPTO_ALTERNATE_KEY_ALGORITHM_CONF_KEY);
486       if (alternateAlgorithm != null) {
487         if (LOG.isDebugEnabled()) {
488           LOG.debug("Unable to decrypt data with current cipher algorithm '"
489               + conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES)
490               + "'. Trying with the alternate cipher algorithm '" + alternateAlgorithm
491               + "' configured.");
492         }
493         Cipher alterCipher = Encryption.getCipher(conf, alternateAlgorithm);
494         if (alterCipher == null) {
495           throw new RuntimeException("Cipher '" + alternateAlgorithm + "' not available");
496         }
497         d = alterCipher.getDecryptor();
498         d.setKey(key);
499         d.setIv(iv); // can be null
500         decrypt(out, in, outLen, d);
501       } else {
502         throw new IOException(e);
503       }
504     }
505   }
506 
507   private static ClassLoader getClassLoaderForClass(Class<?> c) {
508     ClassLoader cl = Thread.currentThread().getContextClassLoader();
509     if (cl == null) {
510       cl = c.getClassLoader();
511     }
512     if (cl == null) {
513       cl = ClassLoader.getSystemClassLoader();
514     }
515     if (cl == null) {
516       throw new RuntimeException("A ClassLoader to load the Cipher could not be determined");
517     }
518     return cl;
519   }
520 
521   public static CipherProvider getCipherProvider(Configuration conf) {
522     String providerClassName = conf.get(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY,
523       DefaultCipherProvider.class.getName());
524     try {
525       CipherProvider provider = (CipherProvider)
526         ReflectionUtils.newInstance(getClassLoaderForClass(CipherProvider.class)
527           .loadClass(providerClassName),
528         conf);
529       return provider;
530     } catch (Exception e) {
531       throw new RuntimeException(e);
532     }
533   }
534 
535   static final Map<Pair<String,String>,KeyProvider> keyProviderCache =
536       new ConcurrentHashMap<Pair<String,String>,KeyProvider>();
537 
538   public static KeyProvider getKeyProvider(Configuration conf) {
539     String providerClassName = conf.get(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
540       KeyStoreKeyProvider.class.getName());
541     String providerParameters = conf.get(HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, "");
542     try {
543       Pair<String,String> providerCacheKey = new Pair<String,String>(providerClassName,
544         providerParameters);
545       KeyProvider provider = keyProviderCache.get(providerCacheKey);
546       if (provider != null) {
547         return provider;
548       }
549       provider = (KeyProvider) ReflectionUtils.newInstance(
550         getClassLoaderForClass(KeyProvider.class).loadClass(providerClassName),
551         conf);
552       provider.init(providerParameters);
553       if (LOG.isDebugEnabled()) {
554         LOG.debug("Installed " + providerClassName + " into key provider cache");
555       }
556       keyProviderCache.put(providerCacheKey, provider);
557       return provider;
558     } catch (Exception e) {
559       throw new RuntimeException(e);
560     }
561   }
562 
563   public static void incrementIv(byte[] iv) {
564     incrementIv(iv, 1);
565   }
566 
567   public static void incrementIv(byte[] iv, int v) {
568     int length = iv.length;
569     boolean carry = true;
570     // TODO: Optimize for v > 1, e.g. 16, 32
571     do {
572       for (int i = 0; i < length; i++) {
573         if (carry) {
574           iv[i] = (byte) ((iv[i] + 1) & 0xFF);
575           carry = 0 == iv[i];
576         } else {
577           break;
578         }
579       }
580       v--;
581     } while (v > 0);
582   }
583 
584 }