001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs.http.client; 019 020 import org.apache.hadoop.conf.Configuration; 021 import org.apache.hadoop.fs.ContentSummary; 022 import org.apache.hadoop.fs.FSDataInputStream; 023 import org.apache.hadoop.fs.FSDataOutputStream; 024 import org.apache.hadoop.fs.FileChecksum; 025 import org.apache.hadoop.fs.FileStatus; 026 import org.apache.hadoop.fs.FileSystem; 027 import org.apache.hadoop.fs.Path; 028 import org.apache.hadoop.fs.PositionedReadable; 029 import org.apache.hadoop.fs.Seekable; 030 import org.apache.hadoop.fs.permission.FsPermission; 031 import org.apache.hadoop.security.UserGroupInformation; 032 import org.apache.hadoop.security.authentication.client.AuthenticatedURL; 033 import org.apache.hadoop.security.authentication.client.Authenticator; 034 import org.apache.hadoop.util.Progressable; 035 import org.apache.hadoop.util.ReflectionUtils; 036 import org.apache.hadoop.util.StringUtils; 037 import org.json.simple.JSONArray; 038 import org.json.simple.JSONObject; 039 import org.json.simple.parser.JSONParser; 040 import org.json.simple.parser.ParseException; 041 042 import java.io.BufferedInputStream; 043 import java.io.BufferedOutputStream; 044 import java.io.DataInput; 045 import java.io.DataOutput; 046 import java.io.FileNotFoundException; 047 import java.io.FilterInputStream; 048 import java.io.IOException; 049 import java.io.InputStream; 050 import java.io.InputStreamReader; 051 import java.io.OutputStream; 052 import java.lang.reflect.Constructor; 053 import java.net.HttpURLConnection; 054 import java.net.URI; 055 import java.net.URISyntaxException; 056 import java.net.URL; 057 import java.net.URLEncoder; 058 import java.text.MessageFormat; 059 import java.util.HashMap; 060 import java.util.Map; 061 062 /** 063 * HttpFSServer implementation of the FileSystemAccess FileSystem. 064 * <p/> 065 * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server. 066 */ 067 public class HttpFSFileSystem extends FileSystem { 068 069 public static final String SERVICE_NAME = "/webhdfs"; 070 071 public static final String SERVICE_VERSION = "/v1"; 072 073 public static final String SERVICE_PREFIX = SERVICE_NAME + SERVICE_VERSION; 074 075 public static final String OP_PARAM = "op"; 076 public static final String DO_AS_PARAM = "doas"; 077 public static final String OVERWRITE_PARAM = "overwrite"; 078 public static final String REPLICATION_PARAM = "replication"; 079 public static final String BLOCKSIZE_PARAM = "blocksize"; 080 public static final String PERMISSION_PARAM = "permission"; 081 public static final String DESTINATION_PARAM = "destination"; 082 public static final String RECURSIVE_PARAM = "recursive"; 083 public static final String OWNER_PARAM = "owner"; 084 public static final String GROUP_PARAM = "group"; 085 public static final String MODIFICATION_TIME_PARAM = "modificationtime"; 086 public static final String ACCESS_TIME_PARAM = "accesstime"; 087 public static final String RENEWER_PARAM = "renewer"; 088 089 public static final String DEFAULT_PERMISSION = "default"; 090 091 public static final String RENAME_JSON = "boolean"; 092 093 public static final String DELETE_JSON = "boolean"; 094 095 public static final String MKDIRS_JSON = "boolean"; 096 097 public static final String HOME_DIR_JSON = "Path"; 098 099 public static final String SET_REPLICATION_JSON = "boolean"; 100 101 public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream"; 102 103 public static enum FILE_TYPE { 104 FILE, DIRECTORY, SYMLINK; 105 106 public static FILE_TYPE getType(FileStatus fileStatus) { 107 if (fileStatus.isFile()) { 108 return FILE; 109 } 110 if (fileStatus.isDirectory()) { 111 return DIRECTORY; 112 } 113 if (fileStatus.isSymlink()) { 114 return SYMLINK; 115 } 116 throw new IllegalArgumentException("Could not determine filetype for: " + 117 fileStatus.getPath()); 118 } 119 } 120 121 public static final String FILE_STATUSES_JSON = "FileStatuses"; 122 public static final String FILE_STATUS_JSON = "FileStatus"; 123 public static final String PATH_SUFFIX_JSON = "pathSuffix"; 124 public static final String TYPE_JSON = "type"; 125 public static final String LENGTH_JSON = "length"; 126 public static final String OWNER_JSON = "owner"; 127 public static final String GROUP_JSON = "group"; 128 public static final String PERMISSION_JSON = "permission"; 129 public static final String ACCESS_TIME_JSON = "accessTime"; 130 public static final String MODIFICATION_TIME_JSON = "modificationTime"; 131 public static final String BLOCK_SIZE_JSON = "blockSize"; 132 public static final String REPLICATION_JSON = "replication"; 133 134 public static final String FILE_CHECKSUM_JSON = "FileChecksum"; 135 public static final String CHECKSUM_ALGORITHM_JSON = "algorithm"; 136 public static final String CHECKSUM_BYTES_JSON = "bytes"; 137 public static final String CHECKSUM_LENGTH_JSON = "length"; 138 139 public static final String CONTENT_SUMMARY_JSON = "ContentSummary"; 140 public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount"; 141 public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount"; 142 public static final String CONTENT_SUMMARY_LENGTH_JSON = "length"; 143 public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota"; 144 public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed"; 145 public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota"; 146 147 public static final String DELEGATION_TOKEN_JSON = "Token"; 148 public static final String DELEGATION_TOKEN_URL_STRING_JSON = "urlString"; 149 150 public static final String ERROR_JSON = "RemoteException"; 151 public static final String ERROR_EXCEPTION_JSON = "exception"; 152 public static final String ERROR_CLASSNAME_JSON = "javaClassName"; 153 public static final String ERROR_MESSAGE_JSON = "message"; 154 155 public static final int HTTP_TEMPORARY_REDIRECT = 307; 156 157 158 /** 159 * Get operations. 160 */ 161 public enum GetOpValues { 162 OPEN, GETFILESTATUS, LISTSTATUS, GETHOMEDIRECTORY, GETCONTENTSUMMARY, GETFILECHECKSUM, 163 GETDELEGATIONTOKEN, GETFILEBLOCKLOCATIONS, INSTRUMENTATION 164 } 165 166 /** 167 * Post operations. 168 */ 169 public static enum PostOpValues { 170 APPEND 171 } 172 173 /** 174 * Put operations. 175 */ 176 public static enum PutOpValues { 177 CREATE, MKDIRS, RENAME, SETOWNER, SETPERMISSION, SETREPLICATION, SETTIMES, 178 RENEWDELEGATIONTOKEN, CANCELDELEGATIONTOKEN 179 } 180 181 /** 182 * Delete operations. 183 */ 184 public static enum DeleteOpValues { 185 DELETE 186 } 187 188 private static final String HTTP_GET = "GET"; 189 private static final String HTTP_PUT = "PUT"; 190 private static final String HTTP_POST = "POST"; 191 private static final String HTTP_DELETE = "DELETE"; 192 193 private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); 194 private URI uri; 195 private Path workingDir; 196 private String doAs; 197 198 /** 199 * Convenience method that creates a <code>HttpURLConnection</code> for the 200 * HttpFSServer file system operations. 201 * <p/> 202 * This methods performs and injects any needed authentication credentials 203 * via the {@link #getConnection(URL, String)} method 204 * 205 * @param method the HTTP method. 206 * @param params the query string parameters. 207 * @param path the file path 208 * @param makeQualified if the path should be 'makeQualified' 209 * 210 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, 211 * authenticated and ready to use for the specified path and file system operation. 212 * 213 * @throws IOException thrown if an IO error occurrs. 214 */ 215 private HttpURLConnection getConnection(String method, Map<String, String> params, 216 Path path, boolean makeQualified) throws IOException { 217 params.put(DO_AS_PARAM, doAs); 218 if (makeQualified) { 219 path = makeQualified(path); 220 } 221 URI uri = path.toUri(); 222 StringBuilder sb = new StringBuilder(); 223 sb.append(uri.getScheme()).append("://").append(uri.getAuthority()). 224 append(SERVICE_PREFIX).append(uri.getPath()); 225 226 String separator = "?"; 227 for (Map.Entry<String, String> entry : params.entrySet()) { 228 sb.append(separator).append(entry.getKey()).append("="). 229 append(URLEncoder.encode(entry.getValue(), "UTF8")); 230 separator = "&"; 231 } 232 URL url = new URL(sb.toString()); 233 return getConnection(url, method); 234 } 235 236 /** 237 * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL. 238 * <p/> 239 * This methods performs and injects any needed authentication credentials. 240 * 241 * @param url url to connect to. 242 * @param method the HTTP method. 243 * 244 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for 245 * the specified path and file system operation. 246 * 247 * @throws IOException thrown if an IO error occurrs. 248 */ 249 private HttpURLConnection getConnection(URL url, String method) throws IOException { 250 Class<? extends Authenticator> klass = 251 getConf().getClass("httpfs.authenticator.class", HttpKerberosAuthenticator.class, Authenticator.class); 252 Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf()); 253 try { 254 HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken); 255 conn.setRequestMethod(method); 256 if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) { 257 conn.setDoOutput(true); 258 } 259 return conn; 260 } catch (Exception ex) { 261 throw new IOException(ex); 262 } 263 } 264 265 /** 266 * Convenience method that JSON Parses the <code>InputStream</code> of a <code>HttpURLConnection</code>. 267 * 268 * @param conn the <code>HttpURLConnection</code>. 269 * 270 * @return the parsed JSON object. 271 * 272 * @throws IOException thrown if the <code>InputStream</code> could not be JSON parsed. 273 */ 274 private static Object jsonParse(HttpURLConnection conn) throws IOException { 275 try { 276 JSONParser parser = new JSONParser(); 277 return parser.parse(new InputStreamReader(conn.getInputStream())); 278 } catch (ParseException ex) { 279 throw new IOException("JSON parser error, " + ex.getMessage(), ex); 280 } 281 } 282 283 /** 284 * Validates the status of an <code>HttpURLConnection</code> against an expected HTTP 285 * status code. If the current status code is not the expected one it throws an exception 286 * with a detail message using Server side error messages if available. 287 * 288 * @param conn the <code>HttpURLConnection</code>. 289 * @param expected the expected HTTP status code. 290 * 291 * @throws IOException thrown if the current status code does not match the expected one. 292 */ 293 private static void validateResponse(HttpURLConnection conn, int expected) throws IOException { 294 int status = conn.getResponseCode(); 295 if (status != expected) { 296 try { 297 JSONObject json = (JSONObject) jsonParse(conn); 298 json = (JSONObject) json.get(ERROR_JSON); 299 String message = (String) json.get(ERROR_MESSAGE_JSON); 300 String exception = (String) json.get(ERROR_EXCEPTION_JSON); 301 String className = (String) json.get(ERROR_CLASSNAME_JSON); 302 303 try { 304 ClassLoader cl = HttpFSFileSystem.class.getClassLoader(); 305 Class klass = cl.loadClass(className); 306 Constructor constr = klass.getConstructor(String.class); 307 throw (IOException) constr.newInstance(message); 308 } catch (IOException ex) { 309 throw ex; 310 } catch (Exception ex) { 311 throw new IOException(MessageFormat.format("{0} - {1}", exception, message)); 312 } 313 } catch (IOException ex) { 314 if (ex.getCause() instanceof IOException) { 315 throw (IOException) ex.getCause(); 316 } 317 throw new IOException(MessageFormat.format("HTTP status [{0}], {1}", status, conn.getResponseMessage())); 318 } 319 } 320 } 321 322 /** 323 * Called after a new FileSystem instance is constructed. 324 * 325 * @param name a uri whose authority section names the host, port, etc. for this FileSystem 326 * @param conf the configuration 327 */ 328 @Override 329 public void initialize(URI name, Configuration conf) throws IOException { 330 UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); 331 doAs = ugi.getUserName(); 332 super.initialize(name, conf); 333 try { 334 uri = new URI(name.getScheme() + "://" + name.getHost() + ":" + name.getPort()); 335 } catch (URISyntaxException ex) { 336 throw new IOException(ex); 337 } 338 } 339 340 /** 341 * Returns a URI whose scheme and authority identify this FileSystem. 342 * 343 * @return the URI whose scheme and authority identify this FileSystem. 344 */ 345 @Override 346 public URI getUri() { 347 return uri; 348 } 349 350 /** 351 * HttpFSServer subclass of the <code>FSDataInputStream</code>. 352 * <p/> 353 * This implementation does not support the 354 * <code>PositionReadable</code> and <code>Seekable</code> methods. 355 */ 356 private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable { 357 358 protected HttpFSDataInputStream(InputStream in, int bufferSize) { 359 super(new BufferedInputStream(in, bufferSize)); 360 } 361 362 @Override 363 public int read(long position, byte[] buffer, int offset, int length) throws IOException { 364 throw new UnsupportedOperationException(); 365 } 366 367 @Override 368 public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { 369 throw new UnsupportedOperationException(); 370 } 371 372 @Override 373 public void readFully(long position, byte[] buffer) throws IOException { 374 throw new UnsupportedOperationException(); 375 } 376 377 @Override 378 public void seek(long pos) throws IOException { 379 throw new UnsupportedOperationException(); 380 } 381 382 @Override 383 public long getPos() throws IOException { 384 throw new UnsupportedOperationException(); 385 } 386 387 @Override 388 public boolean seekToNewSource(long targetPos) throws IOException { 389 throw new UnsupportedOperationException(); 390 } 391 } 392 393 /** 394 * Opens an FSDataInputStream at the indicated Path. 395 * </p> 396 * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the 397 * <code>PositionReadable</code> and <code>Seekable</code> methods. 398 * 399 * @param f the file name to open 400 * @param bufferSize the size of the buffer to be used. 401 */ 402 @Override 403 public FSDataInputStream open(Path f, int bufferSize) throws IOException { 404 Map<String, String> params = new HashMap<String, String>(); 405 params.put(OP_PARAM, GetOpValues.OPEN.toString()); 406 HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); 407 validateResponse(conn, HttpURLConnection.HTTP_OK); 408 return new FSDataInputStream(new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); 409 } 410 411 /** 412 * HttpFSServer subclass of the <code>FSDataOutputStream</code>. 413 * <p/> 414 * This implementation closes the underlying HTTP connection validating the Http connection status 415 * at closing time. 416 */ 417 private static class HttpFSDataOutputStream extends FSDataOutputStream { 418 private HttpURLConnection conn; 419 private int closeStatus; 420 421 public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats) 422 throws IOException { 423 super(out, stats); 424 this.conn = conn; 425 this.closeStatus = closeStatus; 426 } 427 428 @Override 429 public void close() throws IOException { 430 try { 431 super.close(); 432 } finally { 433 validateResponse(conn, closeStatus); 434 } 435 } 436 437 } 438 439 /** 440 * Converts a <code>FsPermission</code> to a Unix octal representation. 441 * 442 * @param p the permission. 443 * 444 * @return the Unix string symbolic reprentation. 445 */ 446 public static String permissionToString(FsPermission p) { 447 return (p == null) ? DEFAULT_PERMISSION : Integer.toString(p.toShort(), 8); 448 } 449 450 /* 451 * Common handling for uploading data for create and append operations. 452 */ 453 private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params, 454 int bufferSize, int expectedStatus) throws IOException { 455 HttpURLConnection conn = getConnection(method, params, f, true); 456 conn.setInstanceFollowRedirects(false); 457 boolean exceptionAlreadyHandled = false; 458 try { 459 if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) { 460 exceptionAlreadyHandled = true; 461 String location = conn.getHeaderField("Location"); 462 if (location != null) { 463 conn = getConnection(new URL(location), method); 464 conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE); 465 try { 466 OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize); 467 return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics); 468 } catch (IOException ex) { 469 validateResponse(conn, expectedStatus); 470 throw ex; 471 } 472 } else { 473 validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 474 throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]"); 475 } 476 } else { 477 throw new IOException( 478 MessageFormat.format("Expected HTTP status was [307], received [{0}]", 479 conn.getResponseCode())); 480 } 481 } catch (IOException ex) { 482 if (exceptionAlreadyHandled) { 483 throw ex; 484 } else { 485 validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 486 throw ex; 487 } 488 } 489 } 490 491 492 /** 493 * Opens an FSDataOutputStream at the indicated Path with write-progress 494 * reporting. 495 * <p/> 496 * IMPORTANT: The <code>Progressable</code> parameter is not used. 497 * 498 * @param f the file name to open. 499 * @param permission file permission. 500 * @param overwrite if a file with this name already exists, then if true, 501 * the file will be overwritten, and if false an error will be thrown. 502 * @param bufferSize the size of the buffer to be used. 503 * @param replication required block replication for the file. 504 * @param blockSize block size. 505 * @param progress progressable. 506 * 507 * @throws IOException 508 * @see #setPermission(Path, FsPermission) 509 */ 510 @Override 511 public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, 512 short replication, long blockSize, Progressable progress) throws IOException { 513 Map<String, String> params = new HashMap<String, String>(); 514 params.put(OP_PARAM, PutOpValues.CREATE.toString()); 515 params.put(OVERWRITE_PARAM, Boolean.toString(overwrite)); 516 params.put(REPLICATION_PARAM, Short.toString(replication)); 517 params.put(BLOCKSIZE_PARAM, Long.toString(blockSize)); 518 params.put(PERMISSION_PARAM, permissionToString(permission)); 519 return uploadData(HTTP_PUT, f, params, bufferSize, HttpURLConnection.HTTP_CREATED); 520 } 521 522 523 /** 524 * Append to an existing file (optional operation). 525 * <p/> 526 * IMPORTANT: The <code>Progressable</code> parameter is not used. 527 * 528 * @param f the existing file to be appended. 529 * @param bufferSize the size of the buffer to be used. 530 * @param progress for reporting progress if it is not null. 531 * 532 * @throws IOException 533 */ 534 @Override 535 public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { 536 Map<String, String> params = new HashMap<String, String>(); 537 params.put(OP_PARAM, PostOpValues.APPEND.toString()); 538 return uploadData(HTTP_POST, f, params, bufferSize, HttpURLConnection.HTTP_OK); 539 } 540 541 /** 542 * Renames Path src to Path dst. Can take place on local fs 543 * or remote DFS. 544 */ 545 @Override 546 public boolean rename(Path src, Path dst) throws IOException { 547 Map<String, String> params = new HashMap<String, String>(); 548 params.put(OP_PARAM, PutOpValues.RENAME.toString()); 549 params.put(DESTINATION_PARAM, dst.toString()); 550 HttpURLConnection conn = getConnection(HTTP_PUT, params, src, true); 551 validateResponse(conn, HttpURLConnection.HTTP_OK); 552 JSONObject json = (JSONObject) jsonParse(conn); 553 return (Boolean) json.get(RENAME_JSON); 554 } 555 556 /** 557 * Delete a file. 558 * 559 * @deprecated Use delete(Path, boolean) instead 560 */ 561 @SuppressWarnings({"deprecation"}) 562 @Deprecated 563 @Override 564 public boolean delete(Path f) throws IOException { 565 return delete(f, false); 566 } 567 568 /** 569 * Delete a file. 570 * 571 * @param f the path to delete. 572 * @param recursive if path is a directory and set to 573 * true, the directory is deleted else throws an exception. In 574 * case of a file the recursive can be set to either true or false. 575 * 576 * @return true if delete is successful else false. 577 * 578 * @throws IOException 579 */ 580 @Override 581 public boolean delete(Path f, boolean recursive) throws IOException { 582 Map<String, String> params = new HashMap<String, String>(); 583 params.put(OP_PARAM, DeleteOpValues.DELETE.toString()); 584 params.put(RECURSIVE_PARAM, Boolean.toString(recursive)); 585 HttpURLConnection conn = getConnection(HTTP_DELETE, params, f, true); 586 validateResponse(conn, HttpURLConnection.HTTP_OK); 587 JSONObject json = (JSONObject) jsonParse(conn); 588 return (Boolean) json.get(DELETE_JSON); 589 } 590 591 /** 592 * List the statuses of the files/directories in the given path if the path is 593 * a directory. 594 * 595 * @param f given path 596 * 597 * @return the statuses of the files/directories in the given patch 598 * 599 * @throws IOException 600 */ 601 @Override 602 public FileStatus[] listStatus(Path f) throws IOException { 603 Map<String, String> params = new HashMap<String, String>(); 604 params.put(OP_PARAM, GetOpValues.LISTSTATUS.toString()); 605 HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); 606 validateResponse(conn, HttpURLConnection.HTTP_OK); 607 JSONObject json = (JSONObject) jsonParse(conn); 608 json = (JSONObject) json.get(FILE_STATUSES_JSON); 609 JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON); 610 FileStatus[] array = new FileStatus[jsonArray.size()]; 611 f = makeQualified(f); 612 for (int i = 0; i < jsonArray.size(); i++) { 613 array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i)); 614 } 615 return array; 616 } 617 618 /** 619 * Set the current working directory for the given file system. All relative 620 * paths will be resolved relative to it. 621 * 622 * @param newDir new directory. 623 */ 624 @Override 625 public void setWorkingDirectory(Path newDir) { 626 workingDir = newDir; 627 } 628 629 /** 630 * Get the current working directory for the given file system 631 * 632 * @return the directory pathname 633 */ 634 @Override 635 public Path getWorkingDirectory() { 636 if (workingDir == null) { 637 workingDir = getHomeDirectory(); 638 } 639 return workingDir; 640 } 641 642 /** 643 * Make the given file and all non-existent parents into 644 * directories. Has the semantics of Unix 'mkdir -p'. 645 * Existence of the directory hierarchy is not an error. 646 */ 647 @Override 648 public boolean mkdirs(Path f, FsPermission permission) throws IOException { 649 Map<String, String> params = new HashMap<String, String>(); 650 params.put(OP_PARAM, PutOpValues.MKDIRS.toString()); 651 params.put(PERMISSION_PARAM, permissionToString(permission)); 652 HttpURLConnection conn = getConnection(HTTP_PUT, params, f, true); 653 validateResponse(conn, HttpURLConnection.HTTP_OK); 654 JSONObject json = (JSONObject) jsonParse(conn); 655 return (Boolean) json.get(MKDIRS_JSON); 656 } 657 658 /** 659 * Return a file status object that represents the path. 660 * 661 * @param f The path we want information from 662 * 663 * @return a FileStatus object 664 * 665 * @throws FileNotFoundException when the path does not exist; 666 * IOException see specific implementation 667 */ 668 @Override 669 public FileStatus getFileStatus(Path f) throws IOException { 670 Map<String, String> params = new HashMap<String, String>(); 671 params.put(OP_PARAM, GetOpValues.GETFILESTATUS.toString()); 672 HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); 673 validateResponse(conn, HttpURLConnection.HTTP_OK); 674 JSONObject json = (JSONObject) jsonParse(conn); 675 json = (JSONObject) json.get(FILE_STATUS_JSON); 676 f = makeQualified(f); 677 return createFileStatus(f, json); 678 } 679 680 /** 681 * Return the current user's home directory in this filesystem. 682 * The default implementation returns "/user/$USER/". 683 */ 684 @Override 685 public Path getHomeDirectory() { 686 Map<String, String> params = new HashMap<String, String>(); 687 params.put(OP_PARAM, GetOpValues.GETHOMEDIRECTORY.toString()); 688 try { 689 HttpURLConnection conn = getConnection(HTTP_GET, params, new Path(getUri().toString(), "/"), false); 690 validateResponse(conn, HttpURLConnection.HTTP_OK); 691 JSONObject json = (JSONObject) jsonParse(conn); 692 return new Path((String) json.get(HOME_DIR_JSON)); 693 } catch (IOException ex) { 694 throw new RuntimeException(ex); 695 } 696 } 697 698 /** 699 * Set owner of a path (i.e. a file or a directory). 700 * The parameters username and groupname cannot both be null. 701 * 702 * @param p The path 703 * @param username If it is null, the original username remains unchanged. 704 * @param groupname If it is null, the original groupname remains unchanged. 705 */ 706 @Override 707 public void setOwner(Path p, String username, String groupname) throws IOException { 708 Map<String, String> params = new HashMap<String, String>(); 709 params.put(OP_PARAM, PutOpValues.SETOWNER.toString()); 710 params.put(OWNER_PARAM, username); 711 params.put(GROUP_PARAM, groupname); 712 HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true); 713 validateResponse(conn, HttpURLConnection.HTTP_OK); 714 } 715 716 /** 717 * Set permission of a path. 718 * 719 * @param p path. 720 * @param permission permission. 721 */ 722 @Override 723 public void setPermission(Path p, FsPermission permission) throws IOException { 724 Map<String, String> params = new HashMap<String, String>(); 725 params.put(OP_PARAM, PutOpValues.SETPERMISSION.toString()); 726 params.put(PERMISSION_PARAM, permissionToString(permission)); 727 HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true); 728 validateResponse(conn, HttpURLConnection.HTTP_OK); 729 } 730 731 /** 732 * Set access time of a file 733 * 734 * @param p The path 735 * @param mtime Set the modification time of this file. 736 * The number of milliseconds since Jan 1, 1970. 737 * A value of -1 means that this call should not set modification time. 738 * @param atime Set the access time of this file. 739 * The number of milliseconds since Jan 1, 1970. 740 * A value of -1 means that this call should not set access time. 741 */ 742 @Override 743 public void setTimes(Path p, long mtime, long atime) throws IOException { 744 Map<String, String> params = new HashMap<String, String>(); 745 params.put(OP_PARAM, PutOpValues.SETTIMES.toString()); 746 params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime)); 747 params.put(ACCESS_TIME_PARAM, Long.toString(atime)); 748 HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true); 749 validateResponse(conn, HttpURLConnection.HTTP_OK); 750 } 751 752 /** 753 * Set replication for an existing file. 754 * 755 * @param src file name 756 * @param replication new replication 757 * 758 * @return true if successful; 759 * false if file does not exist or is a directory 760 * 761 * @throws IOException 762 */ 763 @Override 764 public boolean setReplication(Path src, short replication) throws IOException { 765 Map<String, String> params = new HashMap<String, String>(); 766 params.put(OP_PARAM, PutOpValues.SETREPLICATION.toString()); 767 params.put(REPLICATION_PARAM, Short.toString(replication)); 768 HttpURLConnection conn = getConnection(HTTP_PUT, params, src, true); 769 validateResponse(conn, HttpURLConnection.HTTP_OK); 770 JSONObject json = (JSONObject) jsonParse(conn); 771 return (Boolean) json.get(SET_REPLICATION_JSON); 772 } 773 774 /** 775 * Creates a <code>FileStatus</code> object using a JSON file-status payload 776 * received from a HttpFSServer server. 777 * 778 * @param json a JSON file-status payload received from a HttpFSServer server 779 * 780 * @return the corresponding <code>FileStatus</code> 781 */ 782 private FileStatus createFileStatus(Path parent, JSONObject json) { 783 String pathSuffix = (String) json.get(PATH_SUFFIX_JSON); 784 Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix); 785 FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON)); 786 long len = (Long) json.get(LENGTH_JSON); 787 String owner = (String) json.get(OWNER_JSON); 788 String group = (String) json.get(GROUP_JSON); 789 FsPermission permission = 790 new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8)); 791 long aTime = (Long) json.get(ACCESS_TIME_JSON); 792 long mTime = (Long) json.get(MODIFICATION_TIME_JSON); 793 long blockSize = (Long) json.get(BLOCK_SIZE_JSON); 794 short replication = ((Long) json.get(REPLICATION_JSON)).shortValue(); 795 FileStatus fileStatus = null; 796 797 switch (type) { 798 case FILE: 799 case DIRECTORY: 800 fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY), 801 replication, blockSize, mTime, aTime, 802 permission, owner, group, path); 803 break; 804 case SYMLINK: 805 Path symLink = null; 806 fileStatus = new FileStatus(len, false, 807 replication, blockSize, mTime, aTime, 808 permission, owner, group, symLink, 809 path); 810 } 811 return fileStatus; 812 } 813 814 @Override 815 public ContentSummary getContentSummary(Path f) throws IOException { 816 Map<String, String> params = new HashMap<String, String>(); 817 params.put(OP_PARAM, GetOpValues.GETCONTENTSUMMARY.toString()); 818 HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); 819 validateResponse(conn, HttpURLConnection.HTTP_OK); 820 JSONObject json = (JSONObject) ((JSONObject) jsonParse(conn)).get(CONTENT_SUMMARY_JSON); 821 return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON), 822 (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON), 823 (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON), 824 (Long) json.get(CONTENT_SUMMARY_QUOTA_JSON), 825 (Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON), 826 (Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON) 827 ); 828 } 829 830 @Override 831 public FileChecksum getFileChecksum(Path f) throws IOException { 832 Map<String, String> params = new HashMap<String, String>(); 833 params.put(OP_PARAM, GetOpValues.GETFILECHECKSUM.toString()); 834 HttpURLConnection conn = getConnection(HTTP_GET, params, f, true); 835 validateResponse(conn, HttpURLConnection.HTTP_OK); 836 final JSONObject json = (JSONObject) ((JSONObject) jsonParse(conn)).get(FILE_CHECKSUM_JSON); 837 return new FileChecksum() { 838 @Override 839 public String getAlgorithmName() { 840 return (String) json.get(CHECKSUM_ALGORITHM_JSON); 841 } 842 843 @Override 844 public int getLength() { 845 return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue(); 846 } 847 848 @Override 849 public byte[] getBytes() { 850 return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON)); 851 } 852 853 @Override 854 public void write(DataOutput out) throws IOException { 855 throw new UnsupportedOperationException(); 856 } 857 858 @Override 859 public void readFields(DataInput in) throws IOException { 860 throw new UnsupportedOperationException(); 861 } 862 }; 863 } 864 865 }