001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs.http.client; 019 020 import org.apache.hadoop.classification.InterfaceAudience; 021 import org.apache.hadoop.conf.Configuration; 022 import org.apache.hadoop.fs.ContentSummary; 023 import org.apache.hadoop.fs.DelegationTokenRenewer; 024 import org.apache.hadoop.fs.FSDataInputStream; 025 import org.apache.hadoop.fs.FSDataOutputStream; 026 import org.apache.hadoop.fs.FileChecksum; 027 import org.apache.hadoop.fs.FileStatus; 028 import org.apache.hadoop.fs.FileSystem; 029 import org.apache.hadoop.fs.Path; 030 import org.apache.hadoop.fs.PositionedReadable; 031 import org.apache.hadoop.fs.Seekable; 032 import org.apache.hadoop.fs.permission.FsPermission; 033 import org.apache.hadoop.hdfs.DFSConfigKeys; 034 import org.apache.hadoop.net.NetUtils; 035 import org.apache.hadoop.security.UserGroupInformation; 036 import org.apache.hadoop.security.authentication.client.AuthenticatedURL; 037 import org.apache.hadoop.security.authentication.client.Authenticator; 038 import org.apache.hadoop.security.token.Token; 039 import org.apache.hadoop.security.token.TokenIdentifier; 040 import org.apache.hadoop.util.Progressable; 041 import org.apache.hadoop.util.ReflectionUtils; 042 import org.apache.hadoop.util.StringUtils; 043 import org.json.simple.JSONArray; 044 import org.json.simple.JSONObject; 045 046 import java.io.BufferedInputStream; 047 import java.io.BufferedOutputStream; 048 import java.io.DataInput; 049 import java.io.DataOutput; 050 import java.io.FileNotFoundException; 051 import java.io.FilterInputStream; 052 import java.io.IOException; 053 import java.io.InputStream; 054 import java.io.OutputStream; 055 import java.net.HttpURLConnection; 056 import java.net.InetSocketAddress; 057 import java.net.URI; 058 import java.net.URISyntaxException; 059 import java.net.URL; 060 import java.security.PrivilegedExceptionAction; 061 import java.text.MessageFormat; 062 import java.util.HashMap; 063 import java.util.Map; 064 import java.util.concurrent.Callable; 065 066 /** 067 * HttpFSServer implementation of the FileSystemAccess FileSystem. 068 * <p/> 069 * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server. 070 */ 071 @InterfaceAudience.Private 072 public class HttpFSFileSystem extends FileSystem 073 implements DelegationTokenRenewer.Renewable { 074 075 public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME; 076 077 public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION; 078 079 public static final String SCHEME = "webhdfs"; 080 081 public static final String OP_PARAM = "op"; 082 public static final String DO_AS_PARAM = "doas"; 083 public static final String OVERWRITE_PARAM = "overwrite"; 084 public static final String REPLICATION_PARAM = "replication"; 085 public static final String BLOCKSIZE_PARAM = "blocksize"; 086 public static final String PERMISSION_PARAM = "permission"; 087 public static final String DESTINATION_PARAM = "destination"; 088 public static final String RECURSIVE_PARAM = "recursive"; 089 public static final String OWNER_PARAM = "owner"; 090 public static final String GROUP_PARAM = "group"; 091 public static final String MODIFICATION_TIME_PARAM = "modificationtime"; 092 public static final String ACCESS_TIME_PARAM = "accesstime"; 093 094 public static final Short DEFAULT_PERMISSION = 0755; 095 096 public static final String RENAME_JSON = "boolean"; 097 098 public static final String DELETE_JSON = "boolean"; 099 100 public static final String MKDIRS_JSON = "boolean"; 101 102 public static final String HOME_DIR_JSON = "Path"; 103 104 public static final String SET_REPLICATION_JSON = "boolean"; 105 106 public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream"; 107 108 public static enum FILE_TYPE { 109 FILE, DIRECTORY, SYMLINK; 110 111 public static FILE_TYPE getType(FileStatus fileStatus) { 112 if (fileStatus.isFile()) { 113 return FILE; 114 } 115 if (fileStatus.isDirectory()) { 116 return DIRECTORY; 117 } 118 if (fileStatus.isSymlink()) { 119 return SYMLINK; 120 } 121 throw new IllegalArgumentException("Could not determine filetype for: " + 122 fileStatus.getPath()); 123 } 124 } 125 126 public static final String FILE_STATUSES_JSON = "FileStatuses"; 127 public static final String FILE_STATUS_JSON = "FileStatus"; 128 public static final String PATH_SUFFIX_JSON = "pathSuffix"; 129 public static final String TYPE_JSON = "type"; 130 public static final String LENGTH_JSON = "length"; 131 public static final String OWNER_JSON = "owner"; 132 public static final String GROUP_JSON = "group"; 133 public static final String PERMISSION_JSON = "permission"; 134 public static final String ACCESS_TIME_JSON = "accessTime"; 135 public static final String MODIFICATION_TIME_JSON = "modificationTime"; 136 public static final String BLOCK_SIZE_JSON = "blockSize"; 137 public static final String REPLICATION_JSON = "replication"; 138 139 public static final String FILE_CHECKSUM_JSON = "FileChecksum"; 140 public static final String CHECKSUM_ALGORITHM_JSON = "algorithm"; 141 public static final String CHECKSUM_BYTES_JSON = "bytes"; 142 public static final String CHECKSUM_LENGTH_JSON = "length"; 143 144 public static final String CONTENT_SUMMARY_JSON = "ContentSummary"; 145 public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount"; 146 public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount"; 147 public static final String CONTENT_SUMMARY_LENGTH_JSON = "length"; 148 public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota"; 149 public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed"; 150 public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota"; 151 152 public static final String ERROR_JSON = "RemoteException"; 153 public static final String ERROR_EXCEPTION_JSON = "exception"; 154 public static final String ERROR_CLASSNAME_JSON = "javaClassName"; 155 public static final String ERROR_MESSAGE_JSON = "message"; 156 157 public static final int HTTP_TEMPORARY_REDIRECT = 307; 158 159 private static final String HTTP_GET = "GET"; 160 private static final String HTTP_PUT = "PUT"; 161 private static final String HTTP_POST = "POST"; 162 private static final String HTTP_DELETE = "DELETE"; 163 164 @InterfaceAudience.Private 165 public static enum Operation { 166 OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET), 167 GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET), 168 GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET), 169 INSTRUMENTATION(HTTP_GET), 170 APPEND(HTTP_POST), 171 CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT), 172 SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT), 173 DELETE(HTTP_DELETE); 174 175 private String httpMethod; 176 177 Operation(String httpMethod) { 178 this.httpMethod = httpMethod; 179 } 180 181 public String getMethod() { 182 return httpMethod; 183 } 184 185 } 186 187 188 private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); 189 private URI uri; 190 private InetSocketAddress httpFSAddr; 191 private Path workingDir; 192 private UserGroupInformation realUser; 193 private String doAs; 194 private Token<?> delegationToken; 195 196 //This method enables handling UGI doAs with SPNEGO, we have to 197 //fallback to the realuser who logged in with Kerberos credentials 198 private <T> T doAsRealUserIfNecessary(final Callable<T> callable) 199 throws IOException { 200 try { 201 if (realUser.getShortUserName().equals(doAs)) { 202 return callable.call(); 203 } else { 204 return realUser.doAs(new PrivilegedExceptionAction<T>() { 205 @Override 206 public T run() throws Exception { 207 return callable.call(); 208 } 209 }); 210 } 211 } catch (Exception ex) { 212 throw new IOException(ex.toString(), ex); 213 } 214 } 215 216 /** 217 * Convenience method that creates a <code>HttpURLConnection</code> for the 218 * HttpFSServer file system operations. 219 * <p/> 220 * This methods performs and injects any needed authentication credentials 221 * via the {@link #getConnection(URL, String)} method 222 * 223 * @param method the HTTP method. 224 * @param params the query string parameters. 225 * @param path the file path 226 * @param makeQualified if the path should be 'makeQualified' 227 * 228 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, 229 * authenticated and ready to use for the specified path and file system operation. 230 * 231 * @throws IOException thrown if an IO error occurrs. 232 */ 233 private HttpURLConnection getConnection(final String method, 234 Map<String, String> params, Path path, boolean makeQualified) 235 throws IOException { 236 if (!realUser.getShortUserName().equals(doAs)) { 237 params.put(DO_AS_PARAM, doAs); 238 } 239 HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken); 240 if (makeQualified) { 241 path = makeQualified(path); 242 } 243 final URL url = HttpFSUtils.createHttpURL(path, params); 244 return doAsRealUserIfNecessary(new Callable<HttpURLConnection>() { 245 @Override 246 public HttpURLConnection call() throws Exception { 247 return getConnection(url, method); 248 } 249 }); 250 } 251 252 /** 253 * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL. 254 * <p/> 255 * This methods performs and injects any needed authentication credentials. 256 * 257 * @param url url to connect to. 258 * @param method the HTTP method. 259 * 260 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for 261 * the specified path and file system operation. 262 * 263 * @throws IOException thrown if an IO error occurrs. 264 */ 265 private HttpURLConnection getConnection(URL url, String method) throws IOException { 266 Class<? extends Authenticator> klass = 267 getConf().getClass("httpfs.authenticator.class", 268 HttpFSKerberosAuthenticator.class, Authenticator.class); 269 Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf()); 270 try { 271 HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken); 272 conn.setRequestMethod(method); 273 if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) { 274 conn.setDoOutput(true); 275 } 276 return conn; 277 } catch (Exception ex) { 278 throw new IOException(ex); 279 } 280 } 281 282 /** 283 * Called after a new FileSystem instance is constructed. 284 * 285 * @param name a uri whose authority section names the host, port, etc. for this FileSystem 286 * @param conf the configuration 287 */ 288 @Override 289 public void initialize(URI name, Configuration conf) throws IOException { 290 UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); 291 292 //the real use is the one that has the Kerberos credentials needed for 293 //SPNEGO to work 294 realUser = ugi.getRealUser(); 295 if (realUser == null) { 296 realUser = UserGroupInformation.getLoginUser(); 297 } 298 doAs = ugi.getShortUserName(); 299 super.initialize(name, conf); 300 try { 301 uri = new URI(name.getScheme() + "://" + name.getAuthority()); 302 httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString()); 303 } catch (URISyntaxException ex) { 304 throw new IOException(ex); 305 } 306 } 307 308 @Override 309 public String getScheme() { 310 return SCHEME; 311 } 312 313 /** 314 * Returns a URI whose scheme and authority identify this FileSystem. 315 * 316 * @return the URI whose scheme and authority identify this FileSystem. 317 */ 318 @Override 319 public URI getUri() { 320 return uri; 321 } 322 323 /** 324 * Get the default port for this file system. 325 * @return the default port or 0 if there isn't one 326 */ 327 @Override 328 protected int getDefaultPort() { 329 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, 330 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT); 331 } 332 333 /** 334 * HttpFSServer subclass of the <code>FSDataInputStream</code>. 335 * <p/> 336 * This implementation does not support the 337 * <code>PositionReadable</code> and <code>Seekable</code> methods. 338 */ 339 private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable { 340 341 protected HttpFSDataInputStream(InputStream in, int bufferSize) { 342 super(new BufferedInputStream(in, bufferSize)); 343 } 344 345 @Override 346 public int read(long position, byte[] buffer, int offset, int length) throws IOException { 347 throw new UnsupportedOperationException(); 348 } 349 350 @Override 351 public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { 352 throw new UnsupportedOperationException(); 353 } 354 355 @Override 356 public void readFully(long position, byte[] buffer) throws IOException { 357 throw new UnsupportedOperationException(); 358 } 359 360 @Override 361 public void seek(long pos) throws IOException { 362 throw new UnsupportedOperationException(); 363 } 364 365 @Override 366 public long getPos() throws IOException { 367 throw new UnsupportedOperationException(); 368 } 369 370 @Override 371 public boolean seekToNewSource(long targetPos) throws IOException { 372 throw new UnsupportedOperationException(); 373 } 374 } 375 376 /** 377 * Opens an FSDataInputStream at the indicated Path. 378 * </p> 379 * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the 380 * <code>PositionReadable</code> and <code>Seekable</code> methods. 381 * 382 * @param f the file name to open 383 * @param bufferSize the size of the buffer to be used. 384 */ 385 @Override 386 public FSDataInputStream open(Path f, int bufferSize) throws IOException { 387 Map<String, String> params = new HashMap<String, String>(); 388 params.put(OP_PARAM, Operation.OPEN.toString()); 389 HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params, 390 f, true); 391 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 392 return new FSDataInputStream( 393 new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); 394 } 395 396 /** 397 * HttpFSServer subclass of the <code>FSDataOutputStream</code>. 398 * <p/> 399 * This implementation closes the underlying HTTP connection validating the Http connection status 400 * at closing time. 401 */ 402 private static class HttpFSDataOutputStream extends FSDataOutputStream { 403 private HttpURLConnection conn; 404 private int closeStatus; 405 406 public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats) 407 throws IOException { 408 super(out, stats); 409 this.conn = conn; 410 this.closeStatus = closeStatus; 411 } 412 413 @Override 414 public void close() throws IOException { 415 try { 416 super.close(); 417 } finally { 418 HttpFSUtils.validateResponse(conn, closeStatus); 419 } 420 } 421 422 } 423 424 /** 425 * Converts a <code>FsPermission</code> to a Unix octal representation. 426 * 427 * @param p the permission. 428 * 429 * @return the Unix string symbolic reprentation. 430 */ 431 public static String permissionToString(FsPermission p) { 432 return Integer.toString((p == null) ? DEFAULT_PERMISSION : p.toShort(), 8); 433 } 434 435 /* 436 * Common handling for uploading data for create and append operations. 437 */ 438 private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params, 439 int bufferSize, int expectedStatus) throws IOException { 440 HttpURLConnection conn = getConnection(method, params, f, true); 441 conn.setInstanceFollowRedirects(false); 442 boolean exceptionAlreadyHandled = false; 443 try { 444 if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) { 445 exceptionAlreadyHandled = true; 446 String location = conn.getHeaderField("Location"); 447 if (location != null) { 448 conn = getConnection(new URL(location), method); 449 conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE); 450 try { 451 OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize); 452 return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics); 453 } catch (IOException ex) { 454 HttpFSUtils.validateResponse(conn, expectedStatus); 455 throw ex; 456 } 457 } else { 458 HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 459 throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]"); 460 } 461 } else { 462 throw new IOException( 463 MessageFormat.format("Expected HTTP status was [307], received [{0}]", 464 conn.getResponseCode())); 465 } 466 } catch (IOException ex) { 467 if (exceptionAlreadyHandled) { 468 throw ex; 469 } else { 470 HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 471 throw ex; 472 } 473 } 474 } 475 476 477 /** 478 * Opens an FSDataOutputStream at the indicated Path with write-progress 479 * reporting. 480 * <p/> 481 * IMPORTANT: The <code>Progressable</code> parameter is not used. 482 * 483 * @param f the file name to open. 484 * @param permission file permission. 485 * @param overwrite if a file with this name already exists, then if true, 486 * the file will be overwritten, and if false an error will be thrown. 487 * @param bufferSize the size of the buffer to be used. 488 * @param replication required block replication for the file. 489 * @param blockSize block size. 490 * @param progress progressable. 491 * 492 * @throws IOException 493 * @see #setPermission(Path, FsPermission) 494 */ 495 @Override 496 public FSDataOutputStream create(Path f, FsPermission permission, 497 boolean overwrite, int bufferSize, 498 short replication, long blockSize, 499 Progressable progress) throws IOException { 500 Map<String, String> params = new HashMap<String, String>(); 501 params.put(OP_PARAM, Operation.CREATE.toString()); 502 params.put(OVERWRITE_PARAM, Boolean.toString(overwrite)); 503 params.put(REPLICATION_PARAM, Short.toString(replication)); 504 params.put(BLOCKSIZE_PARAM, Long.toString(blockSize)); 505 params.put(PERMISSION_PARAM, permissionToString(permission)); 506 return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize, 507 HttpURLConnection.HTTP_CREATED); 508 } 509 510 511 /** 512 * Append to an existing file (optional operation). 513 * <p/> 514 * IMPORTANT: The <code>Progressable</code> parameter is not used. 515 * 516 * @param f the existing file to be appended. 517 * @param bufferSize the size of the buffer to be used. 518 * @param progress for reporting progress if it is not null. 519 * 520 * @throws IOException 521 */ 522 @Override 523 public FSDataOutputStream append(Path f, int bufferSize, 524 Progressable progress) throws IOException { 525 Map<String, String> params = new HashMap<String, String>(); 526 params.put(OP_PARAM, Operation.APPEND.toString()); 527 return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize, 528 HttpURLConnection.HTTP_OK); 529 } 530 531 /** 532 * Renames Path src to Path dst. Can take place on local fs 533 * or remote DFS. 534 */ 535 @Override 536 public boolean rename(Path src, Path dst) throws IOException { 537 Map<String, String> params = new HashMap<String, String>(); 538 params.put(OP_PARAM, Operation.RENAME.toString()); 539 params.put(DESTINATION_PARAM, dst.toString()); 540 HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(), 541 params, src, true); 542 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 543 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 544 return (Boolean) json.get(RENAME_JSON); 545 } 546 547 /** 548 * Delete a file. 549 * 550 * @deprecated Use delete(Path, boolean) instead 551 */ 552 @SuppressWarnings({"deprecation"}) 553 @Deprecated 554 @Override 555 public boolean delete(Path f) throws IOException { 556 return delete(f, false); 557 } 558 559 /** 560 * Delete a file. 561 * 562 * @param f the path to delete. 563 * @param recursive if path is a directory and set to 564 * true, the directory is deleted else throws an exception. In 565 * case of a file the recursive can be set to either true or false. 566 * 567 * @return true if delete is successful else false. 568 * 569 * @throws IOException 570 */ 571 @Override 572 public boolean delete(Path f, boolean recursive) throws IOException { 573 Map<String, String> params = new HashMap<String, String>(); 574 params.put(OP_PARAM, Operation.DELETE.toString()); 575 params.put(RECURSIVE_PARAM, Boolean.toString(recursive)); 576 HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(), 577 params, f, true); 578 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 579 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 580 return (Boolean) json.get(DELETE_JSON); 581 } 582 583 /** 584 * List the statuses of the files/directories in the given path if the path is 585 * a directory. 586 * 587 * @param f given path 588 * 589 * @return the statuses of the files/directories in the given patch 590 * 591 * @throws IOException 592 */ 593 @Override 594 public FileStatus[] listStatus(Path f) throws IOException { 595 Map<String, String> params = new HashMap<String, String>(); 596 params.put(OP_PARAM, Operation.LISTSTATUS.toString()); 597 HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(), 598 params, f, true); 599 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 600 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 601 json = (JSONObject) json.get(FILE_STATUSES_JSON); 602 JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON); 603 FileStatus[] array = new FileStatus[jsonArray.size()]; 604 f = makeQualified(f); 605 for (int i = 0; i < jsonArray.size(); i++) { 606 array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i)); 607 } 608 return array; 609 } 610 611 /** 612 * Set the current working directory for the given file system. All relative 613 * paths will be resolved relative to it. 614 * 615 * @param newDir new directory. 616 */ 617 @Override 618 public void setWorkingDirectory(Path newDir) { 619 workingDir = newDir; 620 } 621 622 /** 623 * Get the current working directory for the given file system 624 * 625 * @return the directory pathname 626 */ 627 @Override 628 public Path getWorkingDirectory() { 629 if (workingDir == null) { 630 workingDir = getHomeDirectory(); 631 } 632 return workingDir; 633 } 634 635 /** 636 * Make the given file and all non-existent parents into 637 * directories. Has the semantics of Unix 'mkdir -p'. 638 * Existence of the directory hierarchy is not an error. 639 */ 640 @Override 641 public boolean mkdirs(Path f, FsPermission permission) throws IOException { 642 Map<String, String> params = new HashMap<String, String>(); 643 params.put(OP_PARAM, Operation.MKDIRS.toString()); 644 params.put(PERMISSION_PARAM, permissionToString(permission)); 645 HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(), 646 params, f, true); 647 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 648 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 649 return (Boolean) json.get(MKDIRS_JSON); 650 } 651 652 /** 653 * Return a file status object that represents the path. 654 * 655 * @param f The path we want information from 656 * 657 * @return a FileStatus object 658 * 659 * @throws FileNotFoundException when the path does not exist; 660 * IOException see specific implementation 661 */ 662 @Override 663 public FileStatus getFileStatus(Path f) throws IOException { 664 Map<String, String> params = new HashMap<String, String>(); 665 params.put(OP_PARAM, Operation.GETFILESTATUS.toString()); 666 HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(), 667 params, f, true); 668 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 669 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 670 json = (JSONObject) json.get(FILE_STATUS_JSON); 671 f = makeQualified(f); 672 return createFileStatus(f, json); 673 } 674 675 /** 676 * Return the current user's home directory in this filesystem. 677 * The default implementation returns "/user/$USER/". 678 */ 679 @Override 680 public Path getHomeDirectory() { 681 Map<String, String> params = new HashMap<String, String>(); 682 params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString()); 683 try { 684 HttpURLConnection conn = 685 getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params, 686 new Path(getUri().toString(), "/"), false); 687 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 688 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 689 return new Path((String) json.get(HOME_DIR_JSON)); 690 } catch (IOException ex) { 691 throw new RuntimeException(ex); 692 } 693 } 694 695 /** 696 * Set owner of a path (i.e. a file or a directory). 697 * The parameters username and groupname cannot both be null. 698 * 699 * @param p The path 700 * @param username If it is null, the original username remains unchanged. 701 * @param groupname If it is null, the original groupname remains unchanged. 702 */ 703 @Override 704 public void setOwner(Path p, String username, String groupname) 705 throws IOException { 706 Map<String, String> params = new HashMap<String, String>(); 707 params.put(OP_PARAM, Operation.SETOWNER.toString()); 708 params.put(OWNER_PARAM, username); 709 params.put(GROUP_PARAM, groupname); 710 HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(), 711 params, p, true); 712 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 713 } 714 715 /** 716 * Set permission of a path. 717 * 718 * @param p path. 719 * @param permission permission. 720 */ 721 @Override 722 public void setPermission(Path p, FsPermission permission) throws IOException { 723 Map<String, String> params = new HashMap<String, String>(); 724 params.put(OP_PARAM, Operation.SETPERMISSION.toString()); 725 params.put(PERMISSION_PARAM, permissionToString(permission)); 726 HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true); 727 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 728 } 729 730 /** 731 * Set access time of a file 732 * 733 * @param p The path 734 * @param mtime Set the modification time of this file. 735 * The number of milliseconds since Jan 1, 1970. 736 * A value of -1 means that this call should not set modification time. 737 * @param atime Set the access time of this file. 738 * The number of milliseconds since Jan 1, 1970. 739 * A value of -1 means that this call should not set access time. 740 */ 741 @Override 742 public void setTimes(Path p, long mtime, long atime) throws IOException { 743 Map<String, String> params = new HashMap<String, String>(); 744 params.put(OP_PARAM, Operation.SETTIMES.toString()); 745 params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime)); 746 params.put(ACCESS_TIME_PARAM, Long.toString(atime)); 747 HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(), 748 params, p, true); 749 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 750 } 751 752 /** 753 * Set replication for an existing file. 754 * 755 * @param src file name 756 * @param replication new replication 757 * 758 * @return true if successful; 759 * false if file does not exist or is a directory 760 * 761 * @throws IOException 762 */ 763 @Override 764 public boolean setReplication(Path src, short replication) 765 throws IOException { 766 Map<String, String> params = new HashMap<String, String>(); 767 params.put(OP_PARAM, Operation.SETREPLICATION.toString()); 768 params.put(REPLICATION_PARAM, Short.toString(replication)); 769 HttpURLConnection conn = 770 getConnection(Operation.SETREPLICATION.getMethod(), params, src, true); 771 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 772 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 773 return (Boolean) json.get(SET_REPLICATION_JSON); 774 } 775 776 private FileStatus createFileStatus(Path parent, JSONObject json) { 777 String pathSuffix = (String) json.get(PATH_SUFFIX_JSON); 778 Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix); 779 FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON)); 780 long len = (Long) json.get(LENGTH_JSON); 781 String owner = (String) json.get(OWNER_JSON); 782 String group = (String) json.get(GROUP_JSON); 783 FsPermission permission = 784 new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8)); 785 long aTime = (Long) json.get(ACCESS_TIME_JSON); 786 long mTime = (Long) json.get(MODIFICATION_TIME_JSON); 787 long blockSize = (Long) json.get(BLOCK_SIZE_JSON); 788 short replication = ((Long) json.get(REPLICATION_JSON)).shortValue(); 789 FileStatus fileStatus = null; 790 791 switch (type) { 792 case FILE: 793 case DIRECTORY: 794 fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY), 795 replication, blockSize, mTime, aTime, 796 permission, owner, group, path); 797 break; 798 case SYMLINK: 799 Path symLink = null; 800 fileStatus = new FileStatus(len, false, 801 replication, blockSize, mTime, aTime, 802 permission, owner, group, symLink, 803 path); 804 } 805 return fileStatus; 806 } 807 808 @Override 809 public ContentSummary getContentSummary(Path f) throws IOException { 810 Map<String, String> params = new HashMap<String, String>(); 811 params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString()); 812 HttpURLConnection conn = 813 getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true); 814 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 815 JSONObject json = (JSONObject) ((JSONObject) 816 HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON); 817 return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON), 818 (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON), 819 (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON), 820 (Long) json.get(CONTENT_SUMMARY_QUOTA_JSON), 821 (Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON), 822 (Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON) 823 ); 824 } 825 826 @Override 827 public FileChecksum getFileChecksum(Path f) throws IOException { 828 Map<String, String> params = new HashMap<String, String>(); 829 params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString()); 830 HttpURLConnection conn = 831 getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true); 832 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 833 final JSONObject json = (JSONObject) ((JSONObject) 834 HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON); 835 return new FileChecksum() { 836 @Override 837 public String getAlgorithmName() { 838 return (String) json.get(CHECKSUM_ALGORITHM_JSON); 839 } 840 841 @Override 842 public int getLength() { 843 return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue(); 844 } 845 846 @Override 847 public byte[] getBytes() { 848 return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON)); 849 } 850 851 @Override 852 public void write(DataOutput out) throws IOException { 853 throw new UnsupportedOperationException(); 854 } 855 856 @Override 857 public void readFields(DataInput in) throws IOException { 858 throw new UnsupportedOperationException(); 859 } 860 }; 861 } 862 863 864 @Override 865 public Token<?> getDelegationToken(final String renewer) 866 throws IOException { 867 return doAsRealUserIfNecessary(new Callable<Token<?>>() { 868 @Override 869 public Token<?> call() throws Exception { 870 return HttpFSKerberosAuthenticator. 871 getDelegationToken(uri, httpFSAddr, authToken, renewer); 872 } 873 }); 874 } 875 876 public long renewDelegationToken(final Token<?> token) throws IOException { 877 return doAsRealUserIfNecessary(new Callable<Long>() { 878 @Override 879 public Long call() throws Exception { 880 return HttpFSKerberosAuthenticator. 881 renewDelegationToken(uri, authToken, token); 882 } 883 }); 884 } 885 886 public void cancelDelegationToken(final Token<?> token) throws IOException { 887 HttpFSKerberosAuthenticator. 888 cancelDelegationToken(uri, authToken, token); 889 } 890 891 @Override 892 public Token<?> getRenewToken() { 893 return delegationToken; 894 } 895 896 @Override 897 public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) { 898 delegationToken = token; 899 } 900 901 }