001 package org.apache.hadoop.fs.http.server; 002 003 import org.apache.hadoop.fs.ContentSummary; 004 import org.apache.hadoop.fs.FileChecksum; 005 import org.apache.hadoop.fs.FileStatus; 006 import org.apache.hadoop.fs.FileSystem; 007 import org.apache.hadoop.fs.GlobFilter; 008 import org.apache.hadoop.fs.Path; 009 import org.apache.hadoop.fs.PathFilter; 010 import org.apache.hadoop.fs.http.client.HttpFSFileSystem; 011 import org.apache.hadoop.fs.permission.FsPermission; 012 import org.apache.hadoop.io.IOUtils; 013 import org.apache.hadoop.lib.service.FileSystemAccess; 014 import org.json.simple.JSONArray; 015 import org.json.simple.JSONObject; 016 017 import java.io.IOException; 018 import java.io.InputStream; 019 import java.io.OutputStream; 020 import java.util.LinkedHashMap; 021 import java.util.Map; 022 023 /** 024 * FileSystem operation executors used by {@link HttpFSServer}. 025 */ 026 public class FSOperations { 027 028 /** 029 * Converts a Unix permission octal & symbolic representation 030 * (i.e. 655 or -rwxr--r--) into a FileSystemAccess permission. 031 * 032 * @param str Unix permission symbolic representation. 033 * 034 * @return the FileSystemAccess permission. If the given string was 035 * 'default', it returns <code>FsPermission.getDefault()</code>. 036 */ 037 private static FsPermission getPermission(String str) { 038 FsPermission permission; 039 if (str.equals(HttpFSFileSystem.DEFAULT_PERMISSION)) { 040 permission = FsPermission.getDefault(); 041 } else if (str.length() == 3) { 042 permission = new FsPermission(Short.parseShort(str, 8)); 043 } else { 044 permission = FsPermission.valueOf(str); 045 } 046 return permission; 047 } 048 049 @SuppressWarnings({"unchecked", "deprecation"}) 050 private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) { 051 Map json = new LinkedHashMap(); 052 json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName()); 053 json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString()); 054 json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen()); 055 json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner()); 056 json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup()); 057 json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission())); 058 json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime()); 059 json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime()); 060 json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize()); 061 json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication()); 062 return json; 063 } 064 065 /** 066 * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON 067 * object. 068 * 069 * @param status FileSystemAccess file status. 070 * 071 * @return The JSON representation of the file status. 072 */ 073 @SuppressWarnings({"unchecked", "deprecation"}) 074 private static Map fileStatusToJSON(FileStatus status) { 075 Map json = new LinkedHashMap(); 076 json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true)); 077 return json; 078 } 079 080 /** 081 * Converts a <code>FileChecksum</code> object into a JSON array 082 * object. 083 * 084 * @param checksum file checksum. 085 * 086 * @return The JSON representation of the file checksum. 087 */ 088 @SuppressWarnings({"unchecked"}) 089 private static Map fileChecksumToJSON(FileChecksum checksum) { 090 Map json = new LinkedHashMap(); 091 json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName()); 092 json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON, 093 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes())); 094 json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength()); 095 Map response = new LinkedHashMap(); 096 response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json); 097 return response; 098 } 099 100 /** 101 * Converts a <code>ContentSummary</code> object into a JSON array 102 * object. 103 * 104 * @param contentSummary the content summary 105 * 106 * @return The JSON representation of the content summary. 107 */ 108 @SuppressWarnings({"unchecked"}) 109 private static Map contentSummaryToJSON(ContentSummary contentSummary) { 110 Map json = new LinkedHashMap(); 111 json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount()); 112 json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount()); 113 json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength()); 114 json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota()); 115 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed()); 116 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota()); 117 Map response = new LinkedHashMap(); 118 response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json); 119 return response; 120 } 121 122 /** 123 * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array 124 * object. 125 * 126 * @param status FileSystemAccess file status array. 127 * <code>SCHEME://HOST:PORT</code> in the file status. 128 * 129 * @return The JSON representation of the file status array. 130 */ 131 @SuppressWarnings("unchecked") 132 private static Map fileStatusToJSON(FileStatus[] status) { 133 JSONArray json = new JSONArray(); 134 if (status != null) { 135 for (FileStatus s : status) { 136 json.add(fileStatusToJSONRaw(s, false)); 137 } 138 } 139 Map response = new LinkedHashMap(); 140 Map temp = new LinkedHashMap(); 141 temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json); 142 response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp); 143 return response; 144 } 145 146 /** 147 * Converts an object into a Json Map with with one key-value entry. 148 * <p/> 149 * It assumes the given value is either a JSON primitive type or a 150 * <code>JsonAware</code> instance. 151 * 152 * @param name name for the key of the entry. 153 * @param value for the value of the entry. 154 * 155 * @return the JSON representation of the key-value pair. 156 */ 157 @SuppressWarnings("unchecked") 158 private static JSONObject toJSON(String name, Object value) { 159 JSONObject json = new JSONObject(); 160 json.put(name, value); 161 return json; 162 } 163 164 /** 165 * Executor that performs an append FileSystemAccess files system operation. 166 */ 167 public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> { 168 private InputStream is; 169 private Path path; 170 171 /** 172 * Creates an Append executor. 173 * 174 * @param is input stream to append. 175 * @param path path of the file to append. 176 */ 177 public FSAppend(InputStream is, String path) { 178 this.is = is; 179 this.path = new Path(path); 180 } 181 182 /** 183 * Executes the filesystem operation. 184 * 185 * @param fs filesystem instance to use. 186 * 187 * @return void. 188 * 189 * @throws IOException thrown if an IO error occured. 190 */ 191 @Override 192 public Void execute(FileSystem fs) throws IOException { 193 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 194 OutputStream os = fs.append(path, bufferSize); 195 IOUtils.copyBytes(is, os, bufferSize, true); 196 os.close(); 197 return null; 198 } 199 200 } 201 202 /** 203 * Executor that performs a content-summary FileSystemAccess files system operation. 204 */ 205 public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> { 206 private Path path; 207 208 /** 209 * Creates a content-summary executor. 210 * 211 * @param path the path to retrieve the content-summary. 212 */ 213 public FSContentSummary(String path) { 214 this.path = new Path(path); 215 } 216 217 /** 218 * Executes the filesystem operation. 219 * 220 * @param fs filesystem instance to use. 221 * 222 * @return a Map object (JSON friendly) with the content-summary. 223 * 224 * @throws IOException thrown if an IO error occured. 225 */ 226 @Override 227 public Map execute(FileSystem fs) throws IOException { 228 ContentSummary contentSummary = fs.getContentSummary(path); 229 return contentSummaryToJSON(contentSummary); 230 } 231 232 } 233 234 /** 235 * Executor that performs a create FileSystemAccess files system operation. 236 */ 237 public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> { 238 private InputStream is; 239 private Path path; 240 private String permission; 241 private boolean override; 242 private short replication; 243 private long blockSize; 244 245 /** 246 * Creates a Create executor. 247 * 248 * @param is input stream to for the file to create. 249 * @param path path of the file to create. 250 * @param perm permission for the file. 251 * @param override if the file should be overriden if it already exist. 252 * @param repl the replication factor for the file. 253 * @param blockSize the block size for the file. 254 */ 255 public FSCreate(InputStream is, String path, String perm, boolean override, short repl, long blockSize) { 256 this.is = is; 257 this.path = new Path(path); 258 this.permission = perm; 259 this.override = override; 260 this.replication = repl; 261 this.blockSize = blockSize; 262 } 263 264 /** 265 * Executes the filesystem operation. 266 * 267 * @param fs filesystem instance to use. 268 * 269 * @return The URI of the created file. 270 * 271 * @throws IOException thrown if an IO error occured. 272 */ 273 @Override 274 public Void execute(FileSystem fs) throws IOException { 275 if (replication == -1) { 276 replication = fs.getDefaultReplication(); 277 } 278 if (blockSize == -1) { 279 blockSize = fs.getDefaultBlockSize(); 280 } 281 FsPermission fsPermission = getPermission(permission); 282 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 283 OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null); 284 IOUtils.copyBytes(is, os, bufferSize, true); 285 os.close(); 286 return null; 287 } 288 289 } 290 291 /** 292 * Executor that performs a delete FileSystemAccess files system operation. 293 */ 294 public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> { 295 private Path path; 296 private boolean recursive; 297 298 /** 299 * Creates a Delete executor. 300 * 301 * @param path path to delete. 302 * @param recursive if the delete should be recursive or not. 303 */ 304 public FSDelete(String path, boolean recursive) { 305 this.path = new Path(path); 306 this.recursive = recursive; 307 } 308 309 /** 310 * Executes the filesystem operation. 311 * 312 * @param fs filesystem instance to use. 313 * 314 * @return <code>true</code> if the delete operation was successful, 315 * <code>false</code> otherwise. 316 * 317 * @throws IOException thrown if an IO error occured. 318 */ 319 @Override 320 public JSONObject execute(FileSystem fs) throws IOException { 321 boolean deleted = fs.delete(path, recursive); 322 return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted); 323 } 324 325 } 326 327 /** 328 * Executor that performs a file-checksum FileSystemAccess files system operation. 329 */ 330 public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> { 331 private Path path; 332 333 /** 334 * Creates a file-checksum executor. 335 * 336 * @param path the path to retrieve the checksum. 337 */ 338 public FSFileChecksum(String path) { 339 this.path = new Path(path); 340 } 341 342 /** 343 * Executes the filesystem operation. 344 * 345 * @param fs filesystem instance to use. 346 * 347 * @return a Map object (JSON friendly) with the file checksum. 348 * 349 * @throws IOException thrown if an IO error occured. 350 */ 351 @Override 352 public Map execute(FileSystem fs) throws IOException { 353 FileChecksum checksum = fs.getFileChecksum(path); 354 return fileChecksumToJSON(checksum); 355 } 356 357 } 358 359 /** 360 * Executor that performs a file-status FileSystemAccess files system operation. 361 */ 362 public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> { 363 private Path path; 364 365 /** 366 * Creates a file-status executor. 367 * 368 * @param path the path to retrieve the status. 369 */ 370 public FSFileStatus(String path) { 371 this.path = new Path(path); 372 } 373 374 /** 375 * Executes the filesystem operation. 376 * 377 * @param fs filesystem instance to use. 378 * 379 * @return a Map object (JSON friendly) with the file status. 380 * 381 * @throws IOException thrown if an IO error occured. 382 */ 383 @Override 384 public Map execute(FileSystem fs) throws IOException { 385 FileStatus status = fs.getFileStatus(path); 386 return fileStatusToJSON(status); 387 } 388 389 } 390 391 /** 392 * Executor that performs a home-dir FileSystemAccess files system operation. 393 */ 394 public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> { 395 396 /** 397 * Executes the filesystem operation. 398 * 399 * @param fs filesystem instance to use. 400 * 401 * @return a JSON object with the user home directory. 402 * 403 * @throws IOException thrown if an IO error occured. 404 */ 405 @Override 406 @SuppressWarnings("unchecked") 407 public JSONObject execute(FileSystem fs) throws IOException { 408 Path homeDir = fs.getHomeDirectory(); 409 JSONObject json = new JSONObject(); 410 json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath()); 411 return json; 412 } 413 414 } 415 416 /** 417 * Executor that performs a list-status FileSystemAccess files system operation. 418 */ 419 public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter { 420 private Path path; 421 private PathFilter filter; 422 423 /** 424 * Creates a list-status executor. 425 * 426 * @param path the directory to retrieve the status of its contents. 427 * @param filter glob filter to use. 428 * 429 * @throws IOException thrown if the filter expression is incorrect. 430 */ 431 public FSListStatus(String path, String filter) throws IOException { 432 this.path = new Path(path); 433 this.filter = (filter == null) ? this : new GlobFilter(filter); 434 } 435 436 /** 437 * Executes the filesystem operation. 438 * 439 * @param fs filesystem instance to use. 440 * 441 * @return a Map with the file status of the directory 442 * contents. 443 * 444 * @throws IOException thrown if an IO error occured. 445 */ 446 @Override 447 public Map execute(FileSystem fs) throws IOException { 448 FileStatus[] status = fs.listStatus(path, filter); 449 return fileStatusToJSON(status); 450 } 451 452 @Override 453 public boolean accept(Path path) { 454 return true; 455 } 456 457 } 458 459 /** 460 * Executor that performs a mkdirs FileSystemAccess files system operation. 461 */ 462 public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> { 463 464 private Path path; 465 private String permission; 466 467 /** 468 * Creates a mkdirs executor. 469 * 470 * @param path directory path to create. 471 * @param permission permission to use. 472 */ 473 public FSMkdirs(String path, String permission) { 474 this.path = new Path(path); 475 this.permission = permission; 476 } 477 478 /** 479 * Executes the filesystem operation. 480 * 481 * @param fs filesystem instance to use. 482 * 483 * @return <code>true</code> if the mkdirs operation was successful, 484 * <code>false</code> otherwise. 485 * 486 * @throws IOException thrown if an IO error occured. 487 */ 488 @Override 489 public JSONObject execute(FileSystem fs) throws IOException { 490 FsPermission fsPermission = getPermission(permission); 491 boolean mkdirs = fs.mkdirs(path, fsPermission); 492 return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs); 493 } 494 495 } 496 497 /** 498 * Executor that performs a open FileSystemAccess files system operation. 499 */ 500 public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> { 501 private Path path; 502 503 /** 504 * Creates a open executor. 505 * 506 * @param path file to open. 507 */ 508 public FSOpen(String path) { 509 this.path = new Path(path); 510 } 511 512 /** 513 * Executes the filesystem operation. 514 * 515 * @param fs filesystem instance to use. 516 * 517 * @return The inputstream of the file. 518 * 519 * @throws IOException thrown if an IO error occured. 520 */ 521 @Override 522 public InputStream execute(FileSystem fs) throws IOException { 523 int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096); 524 return fs.open(path, bufferSize); 525 } 526 527 } 528 529 /** 530 * Executor that performs a rename FileSystemAccess files system operation. 531 */ 532 public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> { 533 private Path path; 534 private Path toPath; 535 536 /** 537 * Creates a rename executor. 538 * 539 * @param path path to rename. 540 * @param toPath new name. 541 */ 542 public FSRename(String path, String toPath) { 543 this.path = new Path(path); 544 this.toPath = new Path(toPath); 545 } 546 547 /** 548 * Executes the filesystem operation. 549 * 550 * @param fs filesystem instance to use. 551 * 552 * @return <code>true</code> if the rename operation was successful, 553 * <code>false</code> otherwise. 554 * 555 * @throws IOException thrown if an IO error occured. 556 */ 557 @Override 558 public JSONObject execute(FileSystem fs) throws IOException { 559 boolean renamed = fs.rename(path, toPath); 560 return toJSON(HttpFSFileSystem.RENAME_JSON, renamed); 561 } 562 563 } 564 565 /** 566 * Executor that performs a set-owner FileSystemAccess files system operation. 567 */ 568 public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> { 569 private Path path; 570 private String owner; 571 private String group; 572 573 /** 574 * Creates a set-owner executor. 575 * 576 * @param path the path to set the owner. 577 * @param owner owner to set. 578 * @param group group to set. 579 */ 580 public FSSetOwner(String path, String owner, String group) { 581 this.path = new Path(path); 582 this.owner = owner; 583 this.group = group; 584 } 585 586 /** 587 * Executes the filesystem operation. 588 * 589 * @param fs filesystem instance to use. 590 * 591 * @return void. 592 * 593 * @throws IOException thrown if an IO error occured. 594 */ 595 @Override 596 public Void execute(FileSystem fs) throws IOException { 597 fs.setOwner(path, owner, group); 598 return null; 599 } 600 601 } 602 603 /** 604 * Executor that performs a set-permission FileSystemAccess files system operation. 605 */ 606 public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> { 607 608 private Path path; 609 private String permission; 610 611 /** 612 * Creates a set-permission executor. 613 * 614 * @param path path to set the permission. 615 * @param permission permission to set. 616 */ 617 public FSSetPermission(String path, String permission) { 618 this.path = new Path(path); 619 this.permission = permission; 620 } 621 622 /** 623 * Executes the filesystem operation. 624 * 625 * @param fs filesystem instance to use. 626 * 627 * @return void. 628 * 629 * @throws IOException thrown if an IO error occured. 630 */ 631 @Override 632 public Void execute(FileSystem fs) throws IOException { 633 FsPermission fsPermission = getPermission(permission); 634 fs.setPermission(path, fsPermission); 635 return null; 636 } 637 638 } 639 640 /** 641 * Executor that performs a set-replication FileSystemAccess files system operation. 642 */ 643 public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> { 644 private Path path; 645 private short replication; 646 647 /** 648 * Creates a set-replication executor. 649 * 650 * @param path path to set the replication factor. 651 * @param replication replication factor to set. 652 */ 653 public FSSetReplication(String path, short replication) { 654 this.path = new Path(path); 655 this.replication = replication; 656 } 657 658 /** 659 * Executes the filesystem operation. 660 * 661 * @param fs filesystem instance to use. 662 * 663 * @return <code>true</code> if the replication value was set, 664 * <code>false</code> otherwise. 665 * 666 * @throws IOException thrown if an IO error occured. 667 */ 668 @Override 669 @SuppressWarnings("unchecked") 670 public JSONObject execute(FileSystem fs) throws IOException { 671 boolean ret = fs.setReplication(path, replication); 672 JSONObject json = new JSONObject(); 673 json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret); 674 return json; 675 } 676 677 } 678 679 /** 680 * Executor that performs a set-times FileSystemAccess files system operation. 681 */ 682 public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> { 683 private Path path; 684 private long mTime; 685 private long aTime; 686 687 /** 688 * Creates a set-times executor. 689 * 690 * @param path path to set the times. 691 * @param mTime modified time to set. 692 * @param aTime access time to set. 693 */ 694 public FSSetTimes(String path, long mTime, long aTime) { 695 this.path = new Path(path); 696 this.mTime = mTime; 697 this.aTime = aTime; 698 } 699 700 /** 701 * Executes the filesystem operation. 702 * 703 * @param fs filesystem instance to use. 704 * 705 * @return void. 706 * 707 * @throws IOException thrown if an IO error occured. 708 */ 709 @Override 710 public Void execute(FileSystem fs) throws IOException { 711 fs.setTimes(path, mTime, aTime); 712 return null; 713 } 714 715 } 716 717 }