View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.security.access;
20  
21  import com.google.protobuf.RpcCallback;
22  import com.google.protobuf.RpcController;
23  import com.google.protobuf.Service;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.FileStatus;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.FileUtil;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.fs.permission.FsPermission;
34  import org.apache.hadoop.hbase.Coprocessor;
35  import org.apache.hadoop.hbase.CoprocessorEnvironment;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.DoNotRetryIOException;
38  import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver;
39  import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
40  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
41  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
42  import org.apache.hadoop.hbase.ipc.RpcServer;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
46  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.SecureBulkLoadService;
47  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.PrepareBulkLoadRequest;
48  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.PrepareBulkLoadResponse;
49  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.CleanupBulkLoadRequest;
50  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.CleanupBulkLoadResponse;
51  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.SecureBulkLoadHFilesRequest;
52  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.SecureBulkLoadHFilesResponse;
53  import org.apache.hadoop.hbase.regionserver.HRegion;
54  import org.apache.hadoop.hbase.security.SecureBulkLoadUtil;
55  import org.apache.hadoop.hbase.security.User;
56  import org.apache.hadoop.hbase.security.UserProvider;
57  import org.apache.hadoop.hbase.security.token.FsDelegationToken;
58  import org.apache.hadoop.hbase.util.Bytes;
59  import org.apache.hadoop.hbase.util.FSHDFSUtils;
60  import org.apache.hadoop.hbase.util.Methods;
61  import org.apache.hadoop.hbase.util.Pair;
62  import org.apache.hadoop.io.Text;
63  import org.apache.hadoop.security.UserGroupInformation;
64  import org.apache.hadoop.security.token.Token;
65  
66  import java.io.IOException;
67  import java.math.BigInteger;
68  import java.security.PrivilegedAction;
69  import java.security.SecureRandom;
70  import java.util.ArrayList;
71  import java.util.HashMap;
72  import java.util.List;
73  import java.util.Map;
74  
75  /**
76   * Coprocessor service for bulk loads in secure mode.
77   * This coprocessor has to be installed as part of enabling
78   * security in HBase.
79   *
80   * This service addresses two issues:
81   *
82   * 1. Moving files in a secure filesystem wherein the HBase Client
83   * and HBase Server are different filesystem users.
84   * 2. Does moving in a secure manner. Assuming that the filesystem
85   * is POSIX compliant.
86   *
87   * The algorithm is as follows:
88   *
89   * 1. Create an hbase owned staging directory which is
90   * world traversable (711): /hbase/staging
91   * 2. A user writes out data to his secure output directory: /user/foo/data
92   * 3. A call is made to hbase to create a secret staging directory
93   * which globally rwx (777): /user/staging/averylongandrandomdirectoryname
94   * 4. The user moves the data into the random staging directory,
95   * then calls bulkLoadHFiles()
96   *
97   * Like delegation tokens the strength of the security lies in the length
98   * and randomness of the secret directory.
99   *
100  */
101 @InterfaceAudience.Private
102 public class SecureBulkLoadEndpoint extends SecureBulkLoadService
103     implements CoprocessorService, Coprocessor {
104 
105   public static final long VERSION = 0L;
106 
107   //320/5 = 64 characters
108   private static final int RANDOM_WIDTH = 320;
109   private static final int RANDOM_RADIX = 32;
110 
111   private static Log LOG = LogFactory.getLog(SecureBulkLoadEndpoint.class);
112 
113   private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx");
114   private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x");
115 
116   private SecureRandom random;
117   private FileSystem fs;
118   private Configuration conf;
119 
120   //two levels so it doesn't get deleted accidentally
121   //no sticky bit in Hadoop 1.0
122   private Path baseStagingDir;
123 
124   private RegionCoprocessorEnvironment env;
125 
126   private UserProvider userProvider;
127 
128   @Override
129   public void start(CoprocessorEnvironment env) {
130     this.env = (RegionCoprocessorEnvironment)env;
131     random = new SecureRandom();
132     conf = env.getConfiguration();
133     baseStagingDir = SecureBulkLoadUtil.getBaseStagingDir(conf);
134     this.userProvider = UserProvider.instantiate(conf);
135 
136     try {
137       fs = FileSystem.get(conf);
138       fs.mkdirs(baseStagingDir, PERM_HIDDEN);
139       fs.setPermission(baseStagingDir, PERM_HIDDEN);
140       //no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased
141       fs.mkdirs(new Path(baseStagingDir,"DONOTERASE"), PERM_HIDDEN);
142       FileStatus status = fs.getFileStatus(baseStagingDir);
143       if(status == null) {
144         throw new IllegalStateException("Failed to create staging directory");
145       }
146       if(!status.getPermission().equals(PERM_HIDDEN)) {
147         throw new IllegalStateException(
148             "Directory already exists but permissions aren't set to '-rwx--x--x' ");
149       }
150     } catch (IOException e) {
151       throw new IllegalStateException("Failed to get FileSystem instance",e);
152     }
153   }
154 
155   @Override
156   public void stop(CoprocessorEnvironment env) throws IOException {
157   }
158 
159   @Override
160   public void prepareBulkLoad(RpcController controller,
161                                                  PrepareBulkLoadRequest request,
162                                                  RpcCallback<PrepareBulkLoadResponse> done){
163     try {
164       List<BulkLoadObserver> bulkLoadObservers = getBulkLoadObservers();
165 
166       if(bulkLoadObservers != null) {
167         ObserverContext<RegionCoprocessorEnvironment> ctx =
168                                            new ObserverContext<RegionCoprocessorEnvironment>();
169         ctx.prepare(env);
170 
171         for(BulkLoadObserver bulkLoadObserver : bulkLoadObservers) {
172           bulkLoadObserver.prePrepareBulkLoad(ctx, request);
173         }
174       }
175 
176       String bulkToken = createStagingDir(baseStagingDir,
177           getActiveUser(), ProtobufUtil.toTableName(request.getTableName())).toString();
178       done.run(PrepareBulkLoadResponse.newBuilder().setBulkToken(bulkToken).build());
179     } catch (IOException e) {
180       ResponseConverter.setControllerException(controller, e);
181     }
182     done.run(null);
183   }
184 
185   @Override
186   public void cleanupBulkLoad(RpcController controller,
187                               CleanupBulkLoadRequest request,
188                               RpcCallback<CleanupBulkLoadResponse> done) {
189     try {
190       List<BulkLoadObserver> bulkLoadObservers = getBulkLoadObservers();
191 
192       if(bulkLoadObservers != null) {
193         ObserverContext<RegionCoprocessorEnvironment> ctx =
194                                            new ObserverContext<RegionCoprocessorEnvironment>();
195         ctx.prepare(env);
196 
197         for(BulkLoadObserver bulkLoadObserver : bulkLoadObservers) {
198           bulkLoadObserver.preCleanupBulkLoad(ctx, request);
199         }
200       }
201 
202       fs.delete(new Path(request.getBulkToken()), true);
203       done.run(CleanupBulkLoadResponse.newBuilder().build());
204     } catch (IOException e) {
205       ResponseConverter.setControllerException(controller, e);
206     }
207     done.run(null);
208   }
209 
210   @Override
211   public void secureBulkLoadHFiles(RpcController controller,
212                                    SecureBulkLoadHFilesRequest request,
213                                    RpcCallback<SecureBulkLoadHFilesResponse> done) {
214     final List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>();
215     for(ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) {
216       familyPaths.add(new Pair(el.getFamily().toByteArray(),el.getPath()));
217     }
218     
219     Token userToken = null;
220     if (userProvider.isHadoopSecurityEnabled()) {
221       userToken = new Token(request.getFsToken().getIdentifier().toByteArray(), request.getFsToken()
222               .getPassword().toByteArray(), new Text(request.getFsToken().getKind()), new Text(
223               request.getFsToken().getService()));
224     }
225     final String bulkToken = request.getBulkToken();
226     User user = getActiveUser();
227     final UserGroupInformation ugi = user.getUGI();
228     if(userToken != null) {
229       ugi.addToken(userToken);
230     } else if (userProvider.isHadoopSecurityEnabled()) {
231       //we allow this to pass through in "simple" security mode
232       //for mini cluster testing
233       ResponseConverter.setControllerException(controller,
234           new DoNotRetryIOException("User token cannot be null"));
235       done.run(SecureBulkLoadHFilesResponse.newBuilder().setLoaded(false).build());
236       return;
237     }
238 
239     HRegion region = env.getRegion();
240     boolean bypass = false;
241     if (region.getCoprocessorHost() != null) {
242       try {
243         bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
244       } catch (IOException e) {
245         ResponseConverter.setControllerException(controller, e);
246         done.run(SecureBulkLoadHFilesResponse.newBuilder().setLoaded(false).build());
247         return;
248       }
249     }
250     boolean loaded = false;
251     if (!bypass) {
252       // Get the target fs (HBase region server fs) delegation token
253       // Since we have checked the permission via 'preBulkLoadHFile', now let's give
254       // the 'request user' necessary token to operate on the target fs.
255       // After this point the 'doAs' user will hold two tokens, one for the source fs
256       // ('request user'), another for the target fs (HBase region server principal).
257       if (userProvider.isHadoopSecurityEnabled()) {
258         FsDelegationToken targetfsDelegationToken = new FsDelegationToken(userProvider, "renewer");
259         try {
260           targetfsDelegationToken.acquireDelegationToken(fs);
261         } catch (IOException e) {
262           ResponseConverter.setControllerException(controller, e);
263           done.run(SecureBulkLoadHFilesResponse.newBuilder().setLoaded(false).build());
264           return;
265         }
266         Token<?> targetFsToken = targetfsDelegationToken.getUserToken();
267         if (targetFsToken != null
268             && (userToken == null || !targetFsToken.getService().equals(userToken.getService()))) {
269           ugi.addToken(targetFsToken);
270         }
271       }
272 
273       loaded = ugi.doAs(new PrivilegedAction<Boolean>() {
274         @Override
275         public Boolean run() {
276           FileSystem fs = null;
277           try {
278             Configuration conf = env.getConfiguration();
279             fs = FileSystem.get(conf);
280             for(Pair<byte[], String> el: familyPaths) {
281               Path p = new Path(el.getSecond());
282               Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));
283               if(!fs.exists(stageFamily)) {
284                 fs.mkdirs(stageFamily);
285                 fs.setPermission(stageFamily, PERM_ALL_ACCESS);
286               }
287             }
288             //We call bulkLoadHFiles as requesting user
289             //To enable access prior to staging
290             return env.getRegion().bulkLoadHFiles(familyPaths, true,
291                 new SecureBulkLoadListener(fs, bulkToken, conf));
292           } catch (Exception e) {
293             LOG.error("Failed to complete bulk load", e);
294           } finally {
295             if (fs != null) {
296               try {
297                 if (!UserGroupInformation.getLoginUser().equals(ugi)) {
298                   FileSystem.closeAllForUGI(ugi);
299                 }
300               } catch (IOException e) {
301                 LOG.error("Failed to close FileSystem for " + ugi.getUserName(), e);
302               }
303             }
304           }
305           return false;
306         }
307       });
308     }
309     if (region.getCoprocessorHost() != null) {
310       try {
311         loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
312       } catch (IOException e) {
313         ResponseConverter.setControllerException(controller, e);
314         done.run(SecureBulkLoadHFilesResponse.newBuilder().setLoaded(false).build());
315         return;
316       }
317     }
318     done.run(SecureBulkLoadHFilesResponse.newBuilder().setLoaded(loaded).build());
319   }
320 
321   private List<BulkLoadObserver> getBulkLoadObservers() {
322     List<BulkLoadObserver> coprocessorList =
323               this.env.getRegion().getCoprocessorHost().findCoprocessors(BulkLoadObserver.class);
324 
325     return coprocessorList;
326   }
327 
328   private Path createStagingDir(Path baseDir,
329                                 User user,
330                                 TableName tableName) throws IOException {
331     String tblName = tableName.getNameAsString().replace(":", "_");
332     String randomDir = user.getShortName()+"__"+ tblName +"__"+
333         (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX));
334     return createStagingDir(baseDir, user, randomDir);
335   }
336 
337   private Path createStagingDir(Path baseDir,
338                                 User user,
339                                 String randomDir) throws IOException {
340     Path p = new Path(baseDir, randomDir);
341     fs.mkdirs(p, PERM_ALL_ACCESS);
342     fs.setPermission(p, PERM_ALL_ACCESS);
343     return p;
344   }
345 
346   private User getActiveUser() {
347     User user = RpcServer.getRequestUser();
348     if (user == null) {
349       return null;
350     }
351 
352     //this is for testing
353     if (userProvider.isHadoopSecurityEnabled()
354         && "simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))) {
355       return User.createUserForTesting(conf, user.getShortName(), new String[]{});
356     }
357 
358     return user;
359   }
360 
361   @Override
362   public Service getService() {
363     return this;
364   }
365 
366   private static class SecureBulkLoadListener implements HRegion.BulkLoadListener {
367     // Target filesystem
368     private FileSystem fs;
369     private String stagingDir;
370     private Configuration conf;
371     // Source filesystem
372     private FileSystem srcFs = null;
373     private Map<String, FsPermission> origPermissions = null;
374 
375     public SecureBulkLoadListener(FileSystem fs, String stagingDir, Configuration conf) {
376       this.fs = fs;
377       this.stagingDir = stagingDir;
378       this.conf = conf;
379       this.origPermissions = new HashMap<String, FsPermission>();
380     }
381 
382     @Override
383     public String prepareBulkLoad(final byte[] family, final String srcPath) throws IOException {
384       Path p = new Path(srcPath);
385       Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));
386       if (srcFs == null) {
387         srcFs = FileSystem.get(p.toUri(), conf);
388       }
389 
390       if(!isFile(p)) {
391         throw new IOException("Path does not reference a file: " + p);
392       }
393 
394       // Check to see if the source and target filesystems are the same
395       if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {
396         LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than " +
397             "the destination filesystem. Copying file over to destination staging dir.");
398         FileUtil.copy(srcFs, p, fs, stageP, false, conf);
399       } else {
400         LOG.debug("Moving " + p + " to " + stageP);
401         FileStatus origFileStatus = fs.getFileStatus(p);
402         origPermissions.put(srcPath, origFileStatus.getPermission());
403         if(!fs.rename(p, stageP)) {
404           throw new IOException("Failed to move HFile: " + p + " to " + stageP);
405         }
406       }
407       fs.setPermission(stageP, PERM_ALL_ACCESS);
408       return stageP.toString();
409     }
410 
411     @Override
412     public void doneBulkLoad(byte[] family, String srcPath) throws IOException {
413       LOG.debug("Bulk Load done for: " + srcPath);
414     }
415 
416     @Override
417     public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException {
418       if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {
419         // files are copied so no need to move them back
420         return;
421       }
422       Path p = new Path(srcPath);
423       Path stageP = new Path(stagingDir,
424           new Path(Bytes.toString(family), p.getName()));
425       LOG.debug("Moving " + stageP + " back to " + p);
426       if(!fs.rename(stageP, p))
427         throw new IOException("Failed to move HFile: " + stageP + " to " + p);
428 
429       // restore original permission
430       if (origPermissions.containsKey(srcPath)) {
431         fs.setPermission(p, origPermissions.get(srcPath));
432       } else {
433         LOG.warn("Can't find previous permission for path=" + srcPath);
434       }
435     }
436 
437     /**
438      * Check if the path is referencing a file.
439      * This is mainly needed to avoid symlinks.
440      * @param p
441      * @return true if the p is a file
442      * @throws IOException
443      */
444     private boolean isFile(Path p) throws IOException {
445       FileStatus status = srcFs.getFileStatus(p);
446       boolean isFile = !status.isDir();
447       try {
448         isFile = isFile && !(Boolean)Methods.call(FileStatus.class, status, "isSymlink", null, null);
449       } catch (Exception e) {
450       }
451       return isFile;
452     }
453   }
454 }