001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs.http.client; 019 020 import java.util.ArrayList; 021 import java.util.List; 022 import org.apache.hadoop.classification.InterfaceAudience; 023 import org.apache.hadoop.conf.Configuration; 024 import org.apache.hadoop.fs.ContentSummary; 025 import org.apache.hadoop.fs.DelegationTokenRenewer; 026 import org.apache.hadoop.fs.FSDataInputStream; 027 import org.apache.hadoop.fs.FSDataOutputStream; 028 import org.apache.hadoop.fs.FileChecksum; 029 import org.apache.hadoop.fs.FileStatus; 030 import org.apache.hadoop.fs.FileSystem; 031 import org.apache.hadoop.fs.Path; 032 import org.apache.hadoop.fs.PositionedReadable; 033 import org.apache.hadoop.fs.Seekable; 034 import org.apache.hadoop.fs.permission.FsPermission; 035 import org.apache.hadoop.hdfs.DFSConfigKeys; 036 import org.apache.hadoop.net.NetUtils; 037 import org.apache.hadoop.security.UserGroupInformation; 038 import org.apache.hadoop.security.authentication.client.AuthenticatedURL; 039 import org.apache.hadoop.security.authentication.client.Authenticator; 040 import org.apache.hadoop.security.token.Token; 041 import org.apache.hadoop.security.token.TokenIdentifier; 042 import org.apache.hadoop.util.Progressable; 043 import org.apache.hadoop.util.ReflectionUtils; 044 import org.apache.hadoop.util.StringUtils; 045 import org.json.simple.JSONArray; 046 import org.json.simple.JSONObject; 047 048 import java.io.BufferedInputStream; 049 import java.io.BufferedOutputStream; 050 import java.io.DataInput; 051 import java.io.DataOutput; 052 import java.io.FileNotFoundException; 053 import java.io.FilterInputStream; 054 import java.io.IOException; 055 import java.io.InputStream; 056 import java.io.OutputStream; 057 import java.net.HttpURLConnection; 058 import java.net.InetSocketAddress; 059 import java.net.URI; 060 import java.net.URISyntaxException; 061 import java.net.URL; 062 import java.security.PrivilegedExceptionAction; 063 import java.text.MessageFormat; 064 import java.util.HashMap; 065 import java.util.Map; 066 import java.util.concurrent.Callable; 067 068 /** 069 * HttpFSServer implementation of the FileSystemAccess FileSystem. 070 * <p/> 071 * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server. 072 */ 073 @InterfaceAudience.Private 074 public class HttpFSFileSystem extends FileSystem 075 implements DelegationTokenRenewer.Renewable { 076 077 public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME; 078 079 public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION; 080 081 public static final String SCHEME = "webhdfs"; 082 083 public static final String OP_PARAM = "op"; 084 public static final String DO_AS_PARAM = "doas"; 085 public static final String OVERWRITE_PARAM = "overwrite"; 086 public static final String REPLICATION_PARAM = "replication"; 087 public static final String BLOCKSIZE_PARAM = "blocksize"; 088 public static final String PERMISSION_PARAM = "permission"; 089 public static final String DESTINATION_PARAM = "destination"; 090 public static final String RECURSIVE_PARAM = "recursive"; 091 public static final String SOURCES_PARAM = "sources"; 092 public static final String OWNER_PARAM = "owner"; 093 public static final String GROUP_PARAM = "group"; 094 public static final String MODIFICATION_TIME_PARAM = "modificationtime"; 095 public static final String ACCESS_TIME_PARAM = "accesstime"; 096 097 public static final Short DEFAULT_PERMISSION = 0755; 098 099 public static final String RENAME_JSON = "boolean"; 100 101 public static final String DELETE_JSON = "boolean"; 102 103 public static final String MKDIRS_JSON = "boolean"; 104 105 public static final String HOME_DIR_JSON = "Path"; 106 107 public static final String SET_REPLICATION_JSON = "boolean"; 108 109 public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream"; 110 111 public static enum FILE_TYPE { 112 FILE, DIRECTORY, SYMLINK; 113 114 public static FILE_TYPE getType(FileStatus fileStatus) { 115 if (fileStatus.isFile()) { 116 return FILE; 117 } 118 if (fileStatus.isDirectory()) { 119 return DIRECTORY; 120 } 121 if (fileStatus.isSymlink()) { 122 return SYMLINK; 123 } 124 throw new IllegalArgumentException("Could not determine filetype for: " + 125 fileStatus.getPath()); 126 } 127 } 128 129 public static final String FILE_STATUSES_JSON = "FileStatuses"; 130 public static final String FILE_STATUS_JSON = "FileStatus"; 131 public static final String PATH_SUFFIX_JSON = "pathSuffix"; 132 public static final String TYPE_JSON = "type"; 133 public static final String LENGTH_JSON = "length"; 134 public static final String OWNER_JSON = "owner"; 135 public static final String GROUP_JSON = "group"; 136 public static final String PERMISSION_JSON = "permission"; 137 public static final String ACCESS_TIME_JSON = "accessTime"; 138 public static final String MODIFICATION_TIME_JSON = "modificationTime"; 139 public static final String BLOCK_SIZE_JSON = "blockSize"; 140 public static final String REPLICATION_JSON = "replication"; 141 142 public static final String FILE_CHECKSUM_JSON = "FileChecksum"; 143 public static final String CHECKSUM_ALGORITHM_JSON = "algorithm"; 144 public static final String CHECKSUM_BYTES_JSON = "bytes"; 145 public static final String CHECKSUM_LENGTH_JSON = "length"; 146 147 public static final String CONTENT_SUMMARY_JSON = "ContentSummary"; 148 public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount"; 149 public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount"; 150 public static final String CONTENT_SUMMARY_LENGTH_JSON = "length"; 151 public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota"; 152 public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed"; 153 public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota"; 154 155 public static final String ERROR_JSON = "RemoteException"; 156 public static final String ERROR_EXCEPTION_JSON = "exception"; 157 public static final String ERROR_CLASSNAME_JSON = "javaClassName"; 158 public static final String ERROR_MESSAGE_JSON = "message"; 159 160 public static final int HTTP_TEMPORARY_REDIRECT = 307; 161 162 private static final String HTTP_GET = "GET"; 163 private static final String HTTP_PUT = "PUT"; 164 private static final String HTTP_POST = "POST"; 165 private static final String HTTP_DELETE = "DELETE"; 166 167 @InterfaceAudience.Private 168 public static enum Operation { 169 OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET), 170 GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET), 171 GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET), 172 INSTRUMENTATION(HTTP_GET), 173 APPEND(HTTP_POST), CONCAT(HTTP_POST), 174 CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT), 175 SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT), 176 DELETE(HTTP_DELETE); 177 178 private String httpMethod; 179 180 Operation(String httpMethod) { 181 this.httpMethod = httpMethod; 182 } 183 184 public String getMethod() { 185 return httpMethod; 186 } 187 188 } 189 190 191 private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); 192 private URI uri; 193 private InetSocketAddress httpFSAddr; 194 private Path workingDir; 195 private UserGroupInformation realUser; 196 private String doAs; 197 private Token<?> delegationToken; 198 199 //This method enables handling UGI doAs with SPNEGO, we have to 200 //fallback to the realuser who logged in with Kerberos credentials 201 private <T> T doAsRealUserIfNecessary(final Callable<T> callable) 202 throws IOException { 203 try { 204 if (realUser.getShortUserName().equals(doAs)) { 205 return callable.call(); 206 } else { 207 return realUser.doAs(new PrivilegedExceptionAction<T>() { 208 @Override 209 public T run() throws Exception { 210 return callable.call(); 211 } 212 }); 213 } 214 } catch (Exception ex) { 215 throw new IOException(ex.toString(), ex); 216 } 217 } 218 219 /** 220 * Convenience method that creates a <code>HttpURLConnection</code> for the 221 * HttpFSServer file system operations. 222 * <p/> 223 * This methods performs and injects any needed authentication credentials 224 * via the {@link #getConnection(URL, String)} method 225 * 226 * @param method the HTTP method. 227 * @param params the query string parameters. 228 * @param path the file path 229 * @param makeQualified if the path should be 'makeQualified' 230 * 231 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, 232 * authenticated and ready to use for the specified path and file system operation. 233 * 234 * @throws IOException thrown if an IO error occurrs. 235 */ 236 private HttpURLConnection getConnection(final String method, 237 Map<String, String> params, Path path, boolean makeQualified) 238 throws IOException { 239 if (!realUser.getShortUserName().equals(doAs)) { 240 params.put(DO_AS_PARAM, doAs); 241 } 242 HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken); 243 if (makeQualified) { 244 path = makeQualified(path); 245 } 246 final URL url = HttpFSUtils.createHttpURL(path, params); 247 return doAsRealUserIfNecessary(new Callable<HttpURLConnection>() { 248 @Override 249 public HttpURLConnection call() throws Exception { 250 return getConnection(url, method); 251 } 252 }); 253 } 254 255 /** 256 * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL. 257 * <p/> 258 * This methods performs and injects any needed authentication credentials. 259 * 260 * @param url url to connect to. 261 * @param method the HTTP method. 262 * 263 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for 264 * the specified path and file system operation. 265 * 266 * @throws IOException thrown if an IO error occurrs. 267 */ 268 private HttpURLConnection getConnection(URL url, String method) throws IOException { 269 Class<? extends Authenticator> klass = 270 getConf().getClass("httpfs.authenticator.class", 271 HttpFSKerberosAuthenticator.class, Authenticator.class); 272 Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf()); 273 try { 274 HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken); 275 conn.setRequestMethod(method); 276 if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) { 277 conn.setDoOutput(true); 278 } 279 return conn; 280 } catch (Exception ex) { 281 throw new IOException(ex); 282 } 283 } 284 285 /** 286 * Called after a new FileSystem instance is constructed. 287 * 288 * @param name a uri whose authority section names the host, port, etc. for this FileSystem 289 * @param conf the configuration 290 */ 291 @Override 292 public void initialize(URI name, Configuration conf) throws IOException { 293 UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); 294 295 //the real use is the one that has the Kerberos credentials needed for 296 //SPNEGO to work 297 realUser = ugi.getRealUser(); 298 if (realUser == null) { 299 realUser = UserGroupInformation.getLoginUser(); 300 } 301 doAs = ugi.getShortUserName(); 302 super.initialize(name, conf); 303 try { 304 uri = new URI(name.getScheme() + "://" + name.getAuthority()); 305 httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString()); 306 } catch (URISyntaxException ex) { 307 throw new IOException(ex); 308 } 309 } 310 311 @Override 312 public String getScheme() { 313 return SCHEME; 314 } 315 316 /** 317 * Returns a URI whose scheme and authority identify this FileSystem. 318 * 319 * @return the URI whose scheme and authority identify this FileSystem. 320 */ 321 @Override 322 public URI getUri() { 323 return uri; 324 } 325 326 /** 327 * Get the default port for this file system. 328 * @return the default port or 0 if there isn't one 329 */ 330 @Override 331 protected int getDefaultPort() { 332 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, 333 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT); 334 } 335 336 /** 337 * HttpFSServer subclass of the <code>FSDataInputStream</code>. 338 * <p/> 339 * This implementation does not support the 340 * <code>PositionReadable</code> and <code>Seekable</code> methods. 341 */ 342 private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable { 343 344 protected HttpFSDataInputStream(InputStream in, int bufferSize) { 345 super(new BufferedInputStream(in, bufferSize)); 346 } 347 348 @Override 349 public int read(long position, byte[] buffer, int offset, int length) throws IOException { 350 throw new UnsupportedOperationException(); 351 } 352 353 @Override 354 public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { 355 throw new UnsupportedOperationException(); 356 } 357 358 @Override 359 public void readFully(long position, byte[] buffer) throws IOException { 360 throw new UnsupportedOperationException(); 361 } 362 363 @Override 364 public void seek(long pos) throws IOException { 365 throw new UnsupportedOperationException(); 366 } 367 368 @Override 369 public long getPos() throws IOException { 370 throw new UnsupportedOperationException(); 371 } 372 373 @Override 374 public boolean seekToNewSource(long targetPos) throws IOException { 375 throw new UnsupportedOperationException(); 376 } 377 } 378 379 /** 380 * Opens an FSDataInputStream at the indicated Path. 381 * </p> 382 * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the 383 * <code>PositionReadable</code> and <code>Seekable</code> methods. 384 * 385 * @param f the file name to open 386 * @param bufferSize the size of the buffer to be used. 387 */ 388 @Override 389 public FSDataInputStream open(Path f, int bufferSize) throws IOException { 390 Map<String, String> params = new HashMap<String, String>(); 391 params.put(OP_PARAM, Operation.OPEN.toString()); 392 HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params, 393 f, true); 394 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 395 return new FSDataInputStream( 396 new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); 397 } 398 399 /** 400 * HttpFSServer subclass of the <code>FSDataOutputStream</code>. 401 * <p/> 402 * This implementation closes the underlying HTTP connection validating the Http connection status 403 * at closing time. 404 */ 405 private static class HttpFSDataOutputStream extends FSDataOutputStream { 406 private HttpURLConnection conn; 407 private int closeStatus; 408 409 public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats) 410 throws IOException { 411 super(out, stats); 412 this.conn = conn; 413 this.closeStatus = closeStatus; 414 } 415 416 @Override 417 public void close() throws IOException { 418 try { 419 super.close(); 420 } finally { 421 HttpFSUtils.validateResponse(conn, closeStatus); 422 } 423 } 424 425 } 426 427 /** 428 * Converts a <code>FsPermission</code> to a Unix octal representation. 429 * 430 * @param p the permission. 431 * 432 * @return the Unix string symbolic reprentation. 433 */ 434 public static String permissionToString(FsPermission p) { 435 return Integer.toString((p == null) ? DEFAULT_PERMISSION : p.toShort(), 8); 436 } 437 438 /* 439 * Common handling for uploading data for create and append operations. 440 */ 441 private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params, 442 int bufferSize, int expectedStatus) throws IOException { 443 HttpURLConnection conn = getConnection(method, params, f, true); 444 conn.setInstanceFollowRedirects(false); 445 boolean exceptionAlreadyHandled = false; 446 try { 447 if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) { 448 exceptionAlreadyHandled = true; 449 String location = conn.getHeaderField("Location"); 450 if (location != null) { 451 conn = getConnection(new URL(location), method); 452 conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE); 453 try { 454 OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize); 455 return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics); 456 } catch (IOException ex) { 457 HttpFSUtils.validateResponse(conn, expectedStatus); 458 throw ex; 459 } 460 } else { 461 HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 462 throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]"); 463 } 464 } else { 465 throw new IOException( 466 MessageFormat.format("Expected HTTP status was [307], received [{0}]", 467 conn.getResponseCode())); 468 } 469 } catch (IOException ex) { 470 if (exceptionAlreadyHandled) { 471 throw ex; 472 } else { 473 HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 474 throw ex; 475 } 476 } 477 } 478 479 480 /** 481 * Opens an FSDataOutputStream at the indicated Path with write-progress 482 * reporting. 483 * <p/> 484 * IMPORTANT: The <code>Progressable</code> parameter is not used. 485 * 486 * @param f the file name to open. 487 * @param permission file permission. 488 * @param overwrite if a file with this name already exists, then if true, 489 * the file will be overwritten, and if false an error will be thrown. 490 * @param bufferSize the size of the buffer to be used. 491 * @param replication required block replication for the file. 492 * @param blockSize block size. 493 * @param progress progressable. 494 * 495 * @throws IOException 496 * @see #setPermission(Path, FsPermission) 497 */ 498 @Override 499 public FSDataOutputStream create(Path f, FsPermission permission, 500 boolean overwrite, int bufferSize, 501 short replication, long blockSize, 502 Progressable progress) throws IOException { 503 Map<String, String> params = new HashMap<String, String>(); 504 params.put(OP_PARAM, Operation.CREATE.toString()); 505 params.put(OVERWRITE_PARAM, Boolean.toString(overwrite)); 506 params.put(REPLICATION_PARAM, Short.toString(replication)); 507 params.put(BLOCKSIZE_PARAM, Long.toString(blockSize)); 508 params.put(PERMISSION_PARAM, permissionToString(permission)); 509 return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize, 510 HttpURLConnection.HTTP_CREATED); 511 } 512 513 514 /** 515 * Append to an existing file (optional operation). 516 * <p/> 517 * IMPORTANT: The <code>Progressable</code> parameter is not used. 518 * 519 * @param f the existing file to be appended. 520 * @param bufferSize the size of the buffer to be used. 521 * @param progress for reporting progress if it is not null. 522 * 523 * @throws IOException 524 */ 525 @Override 526 public FSDataOutputStream append(Path f, int bufferSize, 527 Progressable progress) throws IOException { 528 Map<String, String> params = new HashMap<String, String>(); 529 params.put(OP_PARAM, Operation.APPEND.toString()); 530 return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize, 531 HttpURLConnection.HTTP_OK); 532 } 533 534 /** 535 * Concat existing files together. 536 * @param f the path to the target destination. 537 * @param psrcs the paths to the sources to use for the concatenation. 538 * 539 * @throws IOException 540 */ 541 @Override 542 public void concat(Path f, Path[] psrcs) throws IOException { 543 List<String> strPaths = new ArrayList<String>(psrcs.length); 544 for(Path psrc : psrcs) { 545 strPaths.add(psrc.toUri().getPath()); 546 } 547 String srcs = StringUtils.join(",", strPaths); 548 549 Map<String, String> params = new HashMap<String, String>(); 550 params.put(OP_PARAM, Operation.CONCAT.toString()); 551 params.put(SOURCES_PARAM, srcs); 552 HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(), 553 params, f, true); 554 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 555 } 556 557 /** 558 * Renames Path src to Path dst. Can take place on local fs 559 * or remote DFS. 560 */ 561 @Override 562 public boolean rename(Path src, Path dst) throws IOException { 563 Map<String, String> params = new HashMap<String, String>(); 564 params.put(OP_PARAM, Operation.RENAME.toString()); 565 params.put(DESTINATION_PARAM, dst.toString()); 566 HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(), 567 params, src, true); 568 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 569 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 570 return (Boolean) json.get(RENAME_JSON); 571 } 572 573 /** 574 * Delete a file. 575 * 576 * @deprecated Use delete(Path, boolean) instead 577 */ 578 @SuppressWarnings({"deprecation"}) 579 @Deprecated 580 @Override 581 public boolean delete(Path f) throws IOException { 582 return delete(f, false); 583 } 584 585 /** 586 * Delete a file. 587 * 588 * @param f the path to delete. 589 * @param recursive if path is a directory and set to 590 * true, the directory is deleted else throws an exception. In 591 * case of a file the recursive can be set to either true or false. 592 * 593 * @return true if delete is successful else false. 594 * 595 * @throws IOException 596 */ 597 @Override 598 public boolean delete(Path f, boolean recursive) throws IOException { 599 Map<String, String> params = new HashMap<String, String>(); 600 params.put(OP_PARAM, Operation.DELETE.toString()); 601 params.put(RECURSIVE_PARAM, Boolean.toString(recursive)); 602 HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(), 603 params, f, true); 604 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 605 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 606 return (Boolean) json.get(DELETE_JSON); 607 } 608 609 /** 610 * List the statuses of the files/directories in the given path if the path is 611 * a directory. 612 * 613 * @param f given path 614 * 615 * @return the statuses of the files/directories in the given patch 616 * 617 * @throws IOException 618 */ 619 @Override 620 public FileStatus[] listStatus(Path f) throws IOException { 621 Map<String, String> params = new HashMap<String, String>(); 622 params.put(OP_PARAM, Operation.LISTSTATUS.toString()); 623 HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(), 624 params, f, true); 625 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 626 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 627 json = (JSONObject) json.get(FILE_STATUSES_JSON); 628 JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON); 629 FileStatus[] array = new FileStatus[jsonArray.size()]; 630 f = makeQualified(f); 631 for (int i = 0; i < jsonArray.size(); i++) { 632 array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i)); 633 } 634 return array; 635 } 636 637 /** 638 * Set the current working directory for the given file system. All relative 639 * paths will be resolved relative to it. 640 * 641 * @param newDir new directory. 642 */ 643 @Override 644 public void setWorkingDirectory(Path newDir) { 645 workingDir = newDir; 646 } 647 648 /** 649 * Get the current working directory for the given file system 650 * 651 * @return the directory pathname 652 */ 653 @Override 654 public Path getWorkingDirectory() { 655 if (workingDir == null) { 656 workingDir = getHomeDirectory(); 657 } 658 return workingDir; 659 } 660 661 /** 662 * Make the given file and all non-existent parents into 663 * directories. Has the semantics of Unix 'mkdir -p'. 664 * Existence of the directory hierarchy is not an error. 665 */ 666 @Override 667 public boolean mkdirs(Path f, FsPermission permission) throws IOException { 668 Map<String, String> params = new HashMap<String, String>(); 669 params.put(OP_PARAM, Operation.MKDIRS.toString()); 670 params.put(PERMISSION_PARAM, permissionToString(permission)); 671 HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(), 672 params, f, true); 673 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 674 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 675 return (Boolean) json.get(MKDIRS_JSON); 676 } 677 678 /** 679 * Return a file status object that represents the path. 680 * 681 * @param f The path we want information from 682 * 683 * @return a FileStatus object 684 * 685 * @throws FileNotFoundException when the path does not exist; 686 * IOException see specific implementation 687 */ 688 @Override 689 public FileStatus getFileStatus(Path f) throws IOException { 690 Map<String, String> params = new HashMap<String, String>(); 691 params.put(OP_PARAM, Operation.GETFILESTATUS.toString()); 692 HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(), 693 params, f, true); 694 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 695 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 696 json = (JSONObject) json.get(FILE_STATUS_JSON); 697 f = makeQualified(f); 698 return createFileStatus(f, json); 699 } 700 701 /** 702 * Return the current user's home directory in this filesystem. 703 * The default implementation returns "/user/$USER/". 704 */ 705 @Override 706 public Path getHomeDirectory() { 707 Map<String, String> params = new HashMap<String, String>(); 708 params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString()); 709 try { 710 HttpURLConnection conn = 711 getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params, 712 new Path(getUri().toString(), "/"), false); 713 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 714 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 715 return new Path((String) json.get(HOME_DIR_JSON)); 716 } catch (IOException ex) { 717 throw new RuntimeException(ex); 718 } 719 } 720 721 /** 722 * Set owner of a path (i.e. a file or a directory). 723 * The parameters username and groupname cannot both be null. 724 * 725 * @param p The path 726 * @param username If it is null, the original username remains unchanged. 727 * @param groupname If it is null, the original groupname remains unchanged. 728 */ 729 @Override 730 public void setOwner(Path p, String username, String groupname) 731 throws IOException { 732 Map<String, String> params = new HashMap<String, String>(); 733 params.put(OP_PARAM, Operation.SETOWNER.toString()); 734 params.put(OWNER_PARAM, username); 735 params.put(GROUP_PARAM, groupname); 736 HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(), 737 params, p, true); 738 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 739 } 740 741 /** 742 * Set permission of a path. 743 * 744 * @param p path. 745 * @param permission permission. 746 */ 747 @Override 748 public void setPermission(Path p, FsPermission permission) throws IOException { 749 Map<String, String> params = new HashMap<String, String>(); 750 params.put(OP_PARAM, Operation.SETPERMISSION.toString()); 751 params.put(PERMISSION_PARAM, permissionToString(permission)); 752 HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true); 753 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 754 } 755 756 /** 757 * Set access time of a file 758 * 759 * @param p The path 760 * @param mtime Set the modification time of this file. 761 * The number of milliseconds since Jan 1, 1970. 762 * A value of -1 means that this call should not set modification time. 763 * @param atime Set the access time of this file. 764 * The number of milliseconds since Jan 1, 1970. 765 * A value of -1 means that this call should not set access time. 766 */ 767 @Override 768 public void setTimes(Path p, long mtime, long atime) throws IOException { 769 Map<String, String> params = new HashMap<String, String>(); 770 params.put(OP_PARAM, Operation.SETTIMES.toString()); 771 params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime)); 772 params.put(ACCESS_TIME_PARAM, Long.toString(atime)); 773 HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(), 774 params, p, true); 775 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 776 } 777 778 /** 779 * Set replication for an existing file. 780 * 781 * @param src file name 782 * @param replication new replication 783 * 784 * @return true if successful; 785 * false if file does not exist or is a directory 786 * 787 * @throws IOException 788 */ 789 @Override 790 public boolean setReplication(Path src, short replication) 791 throws IOException { 792 Map<String, String> params = new HashMap<String, String>(); 793 params.put(OP_PARAM, Operation.SETREPLICATION.toString()); 794 params.put(REPLICATION_PARAM, Short.toString(replication)); 795 HttpURLConnection conn = 796 getConnection(Operation.SETREPLICATION.getMethod(), params, src, true); 797 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 798 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 799 return (Boolean) json.get(SET_REPLICATION_JSON); 800 } 801 802 private FileStatus createFileStatus(Path parent, JSONObject json) { 803 String pathSuffix = (String) json.get(PATH_SUFFIX_JSON); 804 Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix); 805 FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON)); 806 long len = (Long) json.get(LENGTH_JSON); 807 String owner = (String) json.get(OWNER_JSON); 808 String group = (String) json.get(GROUP_JSON); 809 FsPermission permission = 810 new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8)); 811 long aTime = (Long) json.get(ACCESS_TIME_JSON); 812 long mTime = (Long) json.get(MODIFICATION_TIME_JSON); 813 long blockSize = (Long) json.get(BLOCK_SIZE_JSON); 814 short replication = ((Long) json.get(REPLICATION_JSON)).shortValue(); 815 FileStatus fileStatus = null; 816 817 switch (type) { 818 case FILE: 819 case DIRECTORY: 820 fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY), 821 replication, blockSize, mTime, aTime, 822 permission, owner, group, path); 823 break; 824 case SYMLINK: 825 Path symLink = null; 826 fileStatus = new FileStatus(len, false, 827 replication, blockSize, mTime, aTime, 828 permission, owner, group, symLink, 829 path); 830 } 831 return fileStatus; 832 } 833 834 @Override 835 public ContentSummary getContentSummary(Path f) throws IOException { 836 Map<String, String> params = new HashMap<String, String>(); 837 params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString()); 838 HttpURLConnection conn = 839 getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true); 840 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 841 JSONObject json = (JSONObject) ((JSONObject) 842 HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON); 843 return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON), 844 (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON), 845 (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON), 846 (Long) json.get(CONTENT_SUMMARY_QUOTA_JSON), 847 (Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON), 848 (Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON) 849 ); 850 } 851 852 @Override 853 public FileChecksum getFileChecksum(Path f) throws IOException { 854 Map<String, String> params = new HashMap<String, String>(); 855 params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString()); 856 HttpURLConnection conn = 857 getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true); 858 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 859 final JSONObject json = (JSONObject) ((JSONObject) 860 HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON); 861 return new FileChecksum() { 862 @Override 863 public String getAlgorithmName() { 864 return (String) json.get(CHECKSUM_ALGORITHM_JSON); 865 } 866 867 @Override 868 public int getLength() { 869 return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue(); 870 } 871 872 @Override 873 public byte[] getBytes() { 874 return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON)); 875 } 876 877 @Override 878 public void write(DataOutput out) throws IOException { 879 throw new UnsupportedOperationException(); 880 } 881 882 @Override 883 public void readFields(DataInput in) throws IOException { 884 throw new UnsupportedOperationException(); 885 } 886 }; 887 } 888 889 890 @Override 891 public Token<?> getDelegationToken(final String renewer) 892 throws IOException { 893 return doAsRealUserIfNecessary(new Callable<Token<?>>() { 894 @Override 895 public Token<?> call() throws Exception { 896 return HttpFSKerberosAuthenticator. 897 getDelegationToken(uri, httpFSAddr, authToken, renewer); 898 } 899 }); 900 } 901 902 public long renewDelegationToken(final Token<?> token) throws IOException { 903 return doAsRealUserIfNecessary(new Callable<Long>() { 904 @Override 905 public Long call() throws Exception { 906 return HttpFSKerberosAuthenticator. 907 renewDelegationToken(uri, authToken, token); 908 } 909 }); 910 } 911 912 public void cancelDelegationToken(final Token<?> token) throws IOException { 913 HttpFSKerberosAuthenticator. 914 cancelDelegationToken(uri, authToken, token); 915 } 916 917 @Override 918 public Token<?> getRenewToken() { 919 return delegationToken; 920 } 921 922 @Override 923 public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) { 924 delegationToken = token; 925 } 926 927 }