View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.security.access;
20  
21  import com.google.protobuf.RpcCallback;
22  import com.google.protobuf.RpcController;
23  import com.google.protobuf.Service;
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileStatus;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.fs.permission.FsPermission;
32  import org.apache.hadoop.hbase.Coprocessor;
33  import org.apache.hadoop.hbase.CoprocessorEnvironment;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.DoNotRetryIOException;
36  import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
37  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
38  import org.apache.hadoop.hbase.ipc.RequestContext;
39  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
41  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
42  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.SecureBulkLoadService;
43  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.PrepareBulkLoadRequest;
44  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.PrepareBulkLoadResponse;
45  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.CleanupBulkLoadRequest;
46  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.CleanupBulkLoadResponse;
47  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.SecureBulkLoadHFilesRequest;
48  import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.SecureBulkLoadHFilesResponse;
49  import org.apache.hadoop.hbase.regionserver.HRegion;
50  import org.apache.hadoop.hbase.security.SecureBulkLoadUtil;
51  import org.apache.hadoop.hbase.security.User;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.Methods;
54  import org.apache.hadoop.hbase.util.Pair;
55  import org.apache.hadoop.io.Text;
56  import org.apache.hadoop.security.UserGroupInformation;
57  import org.apache.hadoop.security.token.Token;
58  
59  import java.io.IOException;
60  import java.math.BigInteger;
61  import java.security.PrivilegedAction;
62  import java.security.SecureRandom;
63  import java.util.ArrayList;
64  import java.util.List;
65  
66  /**
67   * Coprocessor service for bulk loads in secure mode.
68   * This coprocessor has to be installed as part of enabling
69   * security in HBase.
70   *
71   * This service addresses two issues:
72   *
73   * 1. Moving files in a secure filesystem wherein the HBase Client
74   * and HBase Server are different filesystem users.
75   * 2. Does moving in a secure manner. Assuming that the filesystem
76   * is POSIX compliant.
77   *
78   * The algorithm is as follows:
79   *
80   * 1. Create an hbase owned staging directory which is
81   * world traversable (711): /hbase/staging
82   * 2. A user writes out data to his secure output directory: /user/foo/data
83   * 3. A call is made to hbase to create a secret staging directory
84   * which globally rwx (777): /user/staging/averylongandrandomdirectoryname
85   * 4. The user makes the data world readable and writable, then moves it
86   * into the random staging directory, then calls bulkLoadHFiles()
87   *
88   * Like delegation tokens the strength of the security lies in the length
89   * and randomness of the secret directory.
90   *
91   */
92  @InterfaceAudience.Private
93  public class SecureBulkLoadEndpoint extends SecureBulkLoadService
94      implements CoprocessorService, Coprocessor {
95  
96    public static final long VERSION = 0L;
97  
98    //320/5 = 64 characters
99    private static final int RANDOM_WIDTH = 320;
100   private static final int RANDOM_RADIX = 32;
101 
102   private static Log LOG = LogFactory.getLog(SecureBulkLoadEndpoint.class);
103 
104   private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx");
105   private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x");
106 
107   private SecureRandom random;
108   private FileSystem fs;
109   private Configuration conf;
110 
111   //two levels so it doesn't get deleted accidentally
112   //no sticky bit in Hadoop 1.0
113   private Path baseStagingDir;
114 
115   private RegionCoprocessorEnvironment env;
116 
117 
118   @Override
119   public void start(CoprocessorEnvironment env) {
120     this.env = (RegionCoprocessorEnvironment)env;
121     random = new SecureRandom();
122     conf = env.getConfiguration();
123     baseStagingDir = SecureBulkLoadUtil.getBaseStagingDir(conf);
124 
125     try {
126       fs = FileSystem.get(conf);
127       fs.mkdirs(baseStagingDir, PERM_HIDDEN);
128       fs.setPermission(baseStagingDir, PERM_HIDDEN);
129       //no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased
130       fs.mkdirs(new Path(baseStagingDir,"DONOTERASE"), PERM_HIDDEN);
131       FileStatus status = fs.getFileStatus(baseStagingDir);
132       if(status == null) {
133         throw new IllegalStateException("Failed to create staging directory");
134       }
135       if(!status.getPermission().equals(PERM_HIDDEN)) {
136         throw new IllegalStateException(
137             "Directory already exists but permissions aren't set to '-rwx--x--x' ");
138       }
139     } catch (IOException e) {
140       throw new IllegalStateException("Failed to get FileSystem instance",e);
141     }
142   }
143 
144   @Override
145   public void stop(CoprocessorEnvironment env) throws IOException {
146   }
147 
148   @Override
149   public void prepareBulkLoad(RpcController controller,
150                                                  PrepareBulkLoadRequest request,
151                                                  RpcCallback<PrepareBulkLoadResponse> done){
152     try {
153       getAccessController().prePrepareBulkLoad(env);
154       String bulkToken = createStagingDir(baseStagingDir,
155           getActiveUser(), ProtobufUtil.toTableName(request.getTableName())).toString();
156       done.run(PrepareBulkLoadResponse.newBuilder().setBulkToken(bulkToken).build());
157     } catch (IOException e) {
158       ResponseConverter.setControllerException(controller, e);
159     }
160     done.run(null);
161   }
162 
163   @Override
164   public void cleanupBulkLoad(RpcController controller,
165                               CleanupBulkLoadRequest request,
166                               RpcCallback<CleanupBulkLoadResponse> done) {
167     try {
168       getAccessController().preCleanupBulkLoad(env);
169       fs.delete(createStagingDir(baseStagingDir,
170           getActiveUser(),
171           env.getRegion().getTableDesc().getTableName(),
172           new Path(request.getBulkToken()).getName()),
173           true);
174       done.run(CleanupBulkLoadResponse.newBuilder().build());
175     } catch (IOException e) {
176       ResponseConverter.setControllerException(controller, e);
177     }
178     done.run(null);
179   }
180 
181   @Override
182   public void secureBulkLoadHFiles(RpcController controller,
183                                    SecureBulkLoadHFilesRequest request,
184                                    RpcCallback<SecureBulkLoadHFilesResponse> done) {
185     final List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>();
186     for(ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) {
187       familyPaths.add(new Pair(el.getFamily().toByteArray(),el.getPath()));
188     }
189     final Token userToken =
190         new Token(request.getFsToken().getIdentifier().toByteArray(),
191                   request.getFsToken().getPassword().toByteArray(),
192                   new Text(request.getFsToken().getKind()),
193                   new Text(request.getFsToken().getService()));
194     final String bulkToken = request.getBulkToken();
195     User user = getActiveUser();
196     final UserGroupInformation ugi = user.getUGI();
197     if(userToken != null) {
198       ugi.addToken(userToken);
199     } else if(User.isSecurityEnabled()) {
200       //we allow this to pass through in "simple" security mode
201       //for mini cluster testing
202       ResponseConverter.setControllerException(controller,
203           new DoNotRetryIOException("User token cannot be null"));
204       return;
205     }
206 
207     HRegion region = env.getRegion();
208     boolean bypass = false;
209     if (region.getCoprocessorHost() != null) {
210       try {
211         bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
212       } catch (IOException e) {
213         ResponseConverter.setControllerException(controller, e);
214         done.run(null);
215         return;
216       }
217     }
218     boolean loaded = false;
219     if (!bypass) {
220       loaded = ugi.doAs(new PrivilegedAction<Boolean>() {
221         @Override
222         public Boolean run() {
223           FileSystem fs = null;
224           try {
225             Configuration conf = env.getConfiguration();
226             fs = FileSystem.get(conf);
227             for(Pair<byte[], String> el: familyPaths) {
228               Path p = new Path(el.getSecond());
229               LOG.trace("Setting permission for: " + p);
230               fs.setPermission(p, PERM_ALL_ACCESS);
231               Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));
232               if(!fs.exists(stageFamily)) {
233                 fs.mkdirs(stageFamily);
234                 fs.setPermission(stageFamily, PERM_ALL_ACCESS);
235               }
236             }
237             //We call bulkLoadHFiles as requesting user
238             //To enable access prior to staging
239             return env.getRegion().bulkLoadHFiles(familyPaths, true,
240                 new SecureBulkLoadListener(fs, bulkToken));
241           } catch (Exception e) {
242             LOG.error("Failed to complete bulk load", e);
243           }
244           return false;
245         }
246       });
247     }
248     if (region.getCoprocessorHost() != null) {
249       try {
250         loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
251       } catch (IOException e) {
252         ResponseConverter.setControllerException(controller, e);
253         done.run(null);
254         return;
255       }
256     }
257     done.run(SecureBulkLoadHFilesResponse.newBuilder().setLoaded(loaded).build());
258   }
259 
260   private AccessController getAccessController() {
261     return (AccessController) this.env.getRegion()
262         .getCoprocessorHost().findCoprocessor(AccessController.class.getName());
263   }
264 
265   private Path createStagingDir(Path baseDir,
266                                 User user,
267                                 TableName tableName) throws IOException {
268     String randomDir = user.getShortName()+"__"+ tableName +"__"+
269         (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX));
270     return createStagingDir(baseDir, user, tableName, randomDir);
271   }
272 
273   private Path createStagingDir(Path baseDir,
274                                 User user,
275                                 TableName tableName,
276                                 String randomDir) throws IOException {
277     Path p = new Path(baseDir, randomDir);
278     fs.mkdirs(p, PERM_ALL_ACCESS);
279     fs.setPermission(p, PERM_ALL_ACCESS);
280     return p;
281   }
282 
283   private User getActiveUser() {
284     User user = RequestContext.getRequestUser();
285     if (!RequestContext.isInRequestContext()) {
286       return null;
287     }
288 
289     //this is for testing
290     if("simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))) {
291       return User.createUserForTesting(conf, user.getShortName(), new String[]{});
292     }
293 
294     return user;
295   }
296 
297   @Override
298   public Service getService() {
299     return this;
300   }
301 
302   private static class SecureBulkLoadListener implements HRegion.BulkLoadListener {
303     private FileSystem fs;
304     private String stagingDir;
305 
306     public SecureBulkLoadListener(FileSystem fs, String stagingDir) {
307       this.fs = fs;
308       this.stagingDir = stagingDir;
309     }
310 
311     @Override
312     public String prepareBulkLoad(final byte[] family, final String srcPath) throws IOException {
313       Path p = new Path(srcPath);
314       Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));
315 
316       if(!isFile(p)) {
317         throw new IOException("Path does not reference a file: " + p);
318       }
319 
320       LOG.debug("Moving " + p + " to " + stageP);
321       if(!fs.rename(p, stageP)) {
322         throw new IOException("Failed to move HFile: " + p + " to " + stageP);
323       }
324       return stageP.toString();
325     }
326 
327     @Override
328     public void doneBulkLoad(byte[] family, String srcPath) throws IOException {
329       LOG.debug("Bulk Load done for: " + srcPath);
330     }
331 
332     @Override
333     public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException {
334       Path p = new Path(srcPath);
335       Path stageP = new Path(stagingDir,
336           new Path(Bytes.toString(family), p.getName()));
337       LOG.debug("Moving " + stageP + " back to " + p);
338       if(!fs.rename(stageP, p))
339         throw new IOException("Failed to move HFile: " + stageP + " to " + p);
340     }
341 
342     /**
343      * Check if the path is referencing a file.
344      * This is mainly needed to avoid symlinks.
345      * @param p
346      * @return true if the p is a file
347      * @throws IOException
348      */
349     private boolean isFile(Path p) throws IOException {
350       FileStatus status = fs.getFileStatus(p);
351       boolean isFile = !status.isDir();
352       try {
353         isFile = isFile && !(Boolean)Methods.call(FileStatus.class, status, "isSymlink", null, null);
354       } catch (Exception e) {
355       }
356       return isFile;
357     }
358   }
359 }