001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs.http.server; 019 020 import org.apache.hadoop.classification.InterfaceAudience; 021 import org.apache.hadoop.fs.ContentSummary; 022 import org.apache.hadoop.fs.FileChecksum; 023 import org.apache.hadoop.fs.FileStatus; 024 import org.apache.hadoop.fs.FileSystem; 025 import org.apache.hadoop.fs.GlobFilter; 026 import org.apache.hadoop.fs.Path; 027 import org.apache.hadoop.fs.PathFilter; 028 import org.apache.hadoop.fs.http.client.HttpFSFileSystem; 029 import org.apache.hadoop.fs.permission.FsPermission; 030 import org.apache.hadoop.io.IOUtils; 031 import org.apache.hadoop.lib.service.FileSystemAccess; 032 import org.json.simple.JSONArray; 033 import org.json.simple.JSONObject; 034 035 import java.io.IOException; 036 import java.io.InputStream; 037 import java.io.OutputStream; 038 import java.util.LinkedHashMap; 039 import java.util.Map; 040 041 /** 042 * FileSystem operation executors used by {@link HttpFSServer}. 043 */ 044 @InterfaceAudience.Private 045 public class FSOperations { 046 047 @SuppressWarnings({"unchecked", "deprecation"}) 048 private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) { 049 Map json = new LinkedHashMap(); 050 json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName()); 051 json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString()); 052 json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen()); 053 json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner()); 054 json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup()); 055 json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission())); 056 json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime()); 057 json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime()); 058 json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize()); 059 json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication()); 060 return json; 061 } 062 063 /** 064 * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON 065 * object. 066 * 067 * @param status FileSystemAccess file status. 068 * 069 * @return The JSON representation of the file status. 070 */ 071 @SuppressWarnings({"unchecked", "deprecation"}) 072 private static Map fileStatusToJSON(FileStatus status) { 073 Map json = new LinkedHashMap(); 074 json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true)); 075 return json; 076 } 077 078 /** 079 * Converts a <code>FileChecksum</code> object into a JSON array 080 * object. 081 * 082 * @param checksum file checksum. 083 * 084 * @return The JSON representation of the file checksum. 085 */ 086 @SuppressWarnings({"unchecked"}) 087 private static Map fileChecksumToJSON(FileChecksum checksum) { 088 Map json = new LinkedHashMap(); 089 json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName()); 090 json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON, 091 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes())); 092 json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength()); 093 Map response = new LinkedHashMap(); 094 response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json); 095 return response; 096 } 097 098 /** 099 * Converts a <code>ContentSummary</code> object into a JSON array 100 * object. 101 * 102 * @param contentSummary the content summary 103 * 104 * @return The JSON representation of the content summary. 105 */ 106 @SuppressWarnings({"unchecked"}) 107 private static Map contentSummaryToJSON(ContentSummary contentSummary) { 108 Map json = new LinkedHashMap(); 109 json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount()); 110 json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount()); 111 json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength()); 112 json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota()); 113 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed()); 114 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota()); 115 Map response = new LinkedHashMap(); 116 response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json); 117 return response; 118 } 119 120 /** 121 * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array 122 * object. 123 * 124 * @param status FileSystemAccess file status array. 125 * <code>SCHEME://HOST:PORT</code> in the file status. 126 * 127 * @return The JSON representation of the file status array. 128 */ 129 @SuppressWarnings("unchecked") 130 private static Map fileStatusToJSON(FileStatus[] status) { 131 JSONArray json = new JSONArray(); 132 if (status != null) { 133 for (FileStatus s : status) { 134 json.add(fileStatusToJSONRaw(s, false)); 135 } 136 } 137 Map response = new LinkedHashMap(); 138 Map temp = new LinkedHashMap(); 139 temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json); 140 response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp); 141 return response; 142 } 143 144 /** 145 * Converts an object into a Json Map with with one key-value entry. 146 * <p/> 147 * It assumes the given value is either a JSON primitive type or a 148 * <code>JsonAware</code> instance. 149 * 150 * @param name name for the key of the entry. 151 * @param value for the value of the entry. 152 * 153 * @return the JSON representation of the key-value pair. 154 */ 155 @SuppressWarnings("unchecked") 156 private static JSONObject toJSON(String name, Object value) { 157 JSONObject json = new JSONObject(); 158 json.put(name, value); 159 return json; 160 } 161 162 /** 163 * Executor that performs an append FileSystemAccess files system operation. 164 */ 165 @InterfaceAudience.Private 166 public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> { 167 private InputStream is; 168 private Path path; 169 170 /** 171 * Creates an Append executor. 172 * 173 * @param is input stream to append. 174 * @param path path of the file to append. 175 */ 176 public FSAppend(InputStream is, String path) { 177 this.is = is; 178 this.path = new Path(path); 179 } 180 181 /** 182 * Executes the filesystem operation. 183 * 184 * @param fs filesystem instance to use. 185 * 186 * @return void. 187 * 188 * @throws IOException thrown if an IO error occured. 189 */ 190 @Override 191 public Void execute(FileSystem fs) throws IOException { 192 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 193 OutputStream os = fs.append(path, bufferSize); 194 IOUtils.copyBytes(is, os, bufferSize, true); 195 os.close(); 196 return null; 197 } 198 199 } 200 201 /** 202 * Executor that performs an append FileSystemAccess files system operation. 203 */ 204 @InterfaceAudience.Private 205 public static class FSConcat implements FileSystemAccess.FileSystemExecutor<Void> { 206 private Path path; 207 private Path[] sources; 208 209 /** 210 * Creates a Concat executor. 211 * 212 * @param path target path to concat to. 213 * @param sources comma seperated absolute paths to use as sources. 214 */ 215 public FSConcat(String path, String[] sources) { 216 this.sources = new Path[sources.length]; 217 218 for(int i = 0; i < sources.length; i++) { 219 this.sources[i] = new Path(sources[i]); 220 } 221 222 this.path = new Path(path); 223 } 224 225 /** 226 * Executes the filesystem operation. 227 * 228 * @param fs filesystem instance to use. 229 * 230 * @return void. 231 * 232 * @throws IOException thrown if an IO error occured. 233 */ 234 @Override 235 public Void execute(FileSystem fs) throws IOException { 236 fs.concat(path, sources); 237 return null; 238 } 239 240 } 241 242 /** 243 * Executor that performs a content-summary FileSystemAccess files system operation. 244 */ 245 @InterfaceAudience.Private 246 public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> { 247 private Path path; 248 249 /** 250 * Creates a content-summary executor. 251 * 252 * @param path the path to retrieve the content-summary. 253 */ 254 public FSContentSummary(String path) { 255 this.path = new Path(path); 256 } 257 258 /** 259 * Executes the filesystem operation. 260 * 261 * @param fs filesystem instance to use. 262 * 263 * @return a Map object (JSON friendly) with the content-summary. 264 * 265 * @throws IOException thrown if an IO error occured. 266 */ 267 @Override 268 public Map execute(FileSystem fs) throws IOException { 269 ContentSummary contentSummary = fs.getContentSummary(path); 270 return contentSummaryToJSON(contentSummary); 271 } 272 273 } 274 275 /** 276 * Executor that performs a create FileSystemAccess files system operation. 277 */ 278 @InterfaceAudience.Private 279 public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> { 280 private InputStream is; 281 private Path path; 282 private short permission; 283 private boolean override; 284 private short replication; 285 private long blockSize; 286 287 /** 288 * Creates a Create executor. 289 * 290 * @param is input stream to for the file to create. 291 * @param path path of the file to create. 292 * @param perm permission for the file. 293 * @param override if the file should be overriden if it already exist. 294 * @param repl the replication factor for the file. 295 * @param blockSize the block size for the file. 296 */ 297 public FSCreate(InputStream is, String path, short perm, boolean override, 298 short repl, long blockSize) { 299 this.is = is; 300 this.path = new Path(path); 301 this.permission = perm; 302 this.override = override; 303 this.replication = repl; 304 this.blockSize = blockSize; 305 } 306 307 /** 308 * Executes the filesystem operation. 309 * 310 * @param fs filesystem instance to use. 311 * 312 * @return The URI of the created file. 313 * 314 * @throws IOException thrown if an IO error occured. 315 */ 316 @Override 317 public Void execute(FileSystem fs) throws IOException { 318 if (replication == -1) { 319 replication = fs.getDefaultReplication(path); 320 } 321 if (blockSize == -1) { 322 blockSize = fs.getDefaultBlockSize(path); 323 } 324 FsPermission fsPermission = new FsPermission(permission); 325 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 326 OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null); 327 IOUtils.copyBytes(is, os, bufferSize, true); 328 os.close(); 329 return null; 330 } 331 332 } 333 334 /** 335 * Executor that performs a delete FileSystemAccess files system operation. 336 */ 337 @InterfaceAudience.Private 338 public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> { 339 private Path path; 340 private boolean recursive; 341 342 /** 343 * Creates a Delete executor. 344 * 345 * @param path path to delete. 346 * @param recursive if the delete should be recursive or not. 347 */ 348 public FSDelete(String path, boolean recursive) { 349 this.path = new Path(path); 350 this.recursive = recursive; 351 } 352 353 /** 354 * Executes the filesystem operation. 355 * 356 * @param fs filesystem instance to use. 357 * 358 * @return <code>true</code> if the delete operation was successful, 359 * <code>false</code> otherwise. 360 * 361 * @throws IOException thrown if an IO error occured. 362 */ 363 @Override 364 public JSONObject execute(FileSystem fs) throws IOException { 365 boolean deleted = fs.delete(path, recursive); 366 return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted); 367 } 368 369 } 370 371 /** 372 * Executor that performs a file-checksum FileSystemAccess files system operation. 373 */ 374 @InterfaceAudience.Private 375 public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> { 376 private Path path; 377 378 /** 379 * Creates a file-checksum executor. 380 * 381 * @param path the path to retrieve the checksum. 382 */ 383 public FSFileChecksum(String path) { 384 this.path = new Path(path); 385 } 386 387 /** 388 * Executes the filesystem operation. 389 * 390 * @param fs filesystem instance to use. 391 * 392 * @return a Map object (JSON friendly) with the file checksum. 393 * 394 * @throws IOException thrown if an IO error occured. 395 */ 396 @Override 397 public Map execute(FileSystem fs) throws IOException { 398 FileChecksum checksum = fs.getFileChecksum(path); 399 return fileChecksumToJSON(checksum); 400 } 401 402 } 403 404 /** 405 * Executor that performs a file-status FileSystemAccess files system operation. 406 */ 407 @InterfaceAudience.Private 408 public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> { 409 private Path path; 410 411 /** 412 * Creates a file-status executor. 413 * 414 * @param path the path to retrieve the status. 415 */ 416 public FSFileStatus(String path) { 417 this.path = new Path(path); 418 } 419 420 /** 421 * Executes the filesystem operation. 422 * 423 * @param fs filesystem instance to use. 424 * 425 * @return a Map object (JSON friendly) with the file status. 426 * 427 * @throws IOException thrown if an IO error occured. 428 */ 429 @Override 430 public Map execute(FileSystem fs) throws IOException { 431 FileStatus status = fs.getFileStatus(path); 432 return fileStatusToJSON(status); 433 } 434 435 } 436 437 /** 438 * Executor that performs a home-dir FileSystemAccess files system operation. 439 */ 440 @InterfaceAudience.Private 441 public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> { 442 443 /** 444 * Executes the filesystem operation. 445 * 446 * @param fs filesystem instance to use. 447 * 448 * @return a JSON object with the user home directory. 449 * 450 * @throws IOException thrown if an IO error occured. 451 */ 452 @Override 453 @SuppressWarnings("unchecked") 454 public JSONObject execute(FileSystem fs) throws IOException { 455 Path homeDir = fs.getHomeDirectory(); 456 JSONObject json = new JSONObject(); 457 json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath()); 458 return json; 459 } 460 461 } 462 463 /** 464 * Executor that performs a list-status FileSystemAccess files system operation. 465 */ 466 @InterfaceAudience.Private 467 public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter { 468 private Path path; 469 private PathFilter filter; 470 471 /** 472 * Creates a list-status executor. 473 * 474 * @param path the directory to retrieve the status of its contents. 475 * @param filter glob filter to use. 476 * 477 * @throws IOException thrown if the filter expression is incorrect. 478 */ 479 public FSListStatus(String path, String filter) throws IOException { 480 this.path = new Path(path); 481 this.filter = (filter == null) ? this : new GlobFilter(filter); 482 } 483 484 /** 485 * Executes the filesystem operation. 486 * 487 * @param fs filesystem instance to use. 488 * 489 * @return a Map with the file status of the directory 490 * contents. 491 * 492 * @throws IOException thrown if an IO error occured. 493 */ 494 @Override 495 public Map execute(FileSystem fs) throws IOException { 496 FileStatus[] status = fs.listStatus(path, filter); 497 return fileStatusToJSON(status); 498 } 499 500 @Override 501 public boolean accept(Path path) { 502 return true; 503 } 504 505 } 506 507 /** 508 * Executor that performs a mkdirs FileSystemAccess files system operation. 509 */ 510 @InterfaceAudience.Private 511 public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> { 512 513 private Path path; 514 private short permission; 515 516 /** 517 * Creates a mkdirs executor. 518 * 519 * @param path directory path to create. 520 * @param permission permission to use. 521 */ 522 public FSMkdirs(String path, short permission) { 523 this.path = new Path(path); 524 this.permission = permission; 525 } 526 527 /** 528 * Executes the filesystem operation. 529 * 530 * @param fs filesystem instance to use. 531 * 532 * @return <code>true</code> if the mkdirs operation was successful, 533 * <code>false</code> otherwise. 534 * 535 * @throws IOException thrown if an IO error occured. 536 */ 537 @Override 538 public JSONObject execute(FileSystem fs) throws IOException { 539 FsPermission fsPermission = new FsPermission(permission); 540 boolean mkdirs = fs.mkdirs(path, fsPermission); 541 return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs); 542 } 543 544 } 545 546 /** 547 * Executor that performs a open FileSystemAccess files system operation. 548 */ 549 @InterfaceAudience.Private 550 public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> { 551 private Path path; 552 553 /** 554 * Creates a open executor. 555 * 556 * @param path file to open. 557 */ 558 public FSOpen(String path) { 559 this.path = new Path(path); 560 } 561 562 /** 563 * Executes the filesystem operation. 564 * 565 * @param fs filesystem instance to use. 566 * 567 * @return The inputstream of the file. 568 * 569 * @throws IOException thrown if an IO error occured. 570 */ 571 @Override 572 public InputStream execute(FileSystem fs) throws IOException { 573 int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096); 574 return fs.open(path, bufferSize); 575 } 576 577 } 578 579 /** 580 * Executor that performs a rename FileSystemAccess files system operation. 581 */ 582 @InterfaceAudience.Private 583 public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> { 584 private Path path; 585 private Path toPath; 586 587 /** 588 * Creates a rename executor. 589 * 590 * @param path path to rename. 591 * @param toPath new name. 592 */ 593 public FSRename(String path, String toPath) { 594 this.path = new Path(path); 595 this.toPath = new Path(toPath); 596 } 597 598 /** 599 * Executes the filesystem operation. 600 * 601 * @param fs filesystem instance to use. 602 * 603 * @return <code>true</code> if the rename operation was successful, 604 * <code>false</code> otherwise. 605 * 606 * @throws IOException thrown if an IO error occured. 607 */ 608 @Override 609 public JSONObject execute(FileSystem fs) throws IOException { 610 boolean renamed = fs.rename(path, toPath); 611 return toJSON(HttpFSFileSystem.RENAME_JSON, renamed); 612 } 613 614 } 615 616 /** 617 * Executor that performs a set-owner FileSystemAccess files system operation. 618 */ 619 @InterfaceAudience.Private 620 public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> { 621 private Path path; 622 private String owner; 623 private String group; 624 625 /** 626 * Creates a set-owner executor. 627 * 628 * @param path the path to set the owner. 629 * @param owner owner to set. 630 * @param group group to set. 631 */ 632 public FSSetOwner(String path, String owner, String group) { 633 this.path = new Path(path); 634 this.owner = owner; 635 this.group = group; 636 } 637 638 /** 639 * Executes the filesystem operation. 640 * 641 * @param fs filesystem instance to use. 642 * 643 * @return void. 644 * 645 * @throws IOException thrown if an IO error occured. 646 */ 647 @Override 648 public Void execute(FileSystem fs) throws IOException { 649 fs.setOwner(path, owner, group); 650 return null; 651 } 652 653 } 654 655 /** 656 * Executor that performs a set-permission FileSystemAccess files system operation. 657 */ 658 @InterfaceAudience.Private 659 public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> { 660 661 private Path path; 662 private short permission; 663 664 /** 665 * Creates a set-permission executor. 666 * 667 * @param path path to set the permission. 668 * @param permission permission to set. 669 */ 670 public FSSetPermission(String path, short permission) { 671 this.path = new Path(path); 672 this.permission = permission; 673 } 674 675 /** 676 * Executes the filesystem operation. 677 * 678 * @param fs filesystem instance to use. 679 * 680 * @return void. 681 * 682 * @throws IOException thrown if an IO error occured. 683 */ 684 @Override 685 public Void execute(FileSystem fs) throws IOException { 686 FsPermission fsPermission = new FsPermission(permission); 687 fs.setPermission(path, fsPermission); 688 return null; 689 } 690 691 } 692 693 /** 694 * Executor that performs a set-replication FileSystemAccess files system operation. 695 */ 696 @InterfaceAudience.Private 697 public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> { 698 private Path path; 699 private short replication; 700 701 /** 702 * Creates a set-replication executor. 703 * 704 * @param path path to set the replication factor. 705 * @param replication replication factor to set. 706 */ 707 public FSSetReplication(String path, short replication) { 708 this.path = new Path(path); 709 this.replication = replication; 710 } 711 712 /** 713 * Executes the filesystem operation. 714 * 715 * @param fs filesystem instance to use. 716 * 717 * @return <code>true</code> if the replication value was set, 718 * <code>false</code> otherwise. 719 * 720 * @throws IOException thrown if an IO error occured. 721 */ 722 @Override 723 @SuppressWarnings("unchecked") 724 public JSONObject execute(FileSystem fs) throws IOException { 725 boolean ret = fs.setReplication(path, replication); 726 JSONObject json = new JSONObject(); 727 json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret); 728 return json; 729 } 730 731 } 732 733 /** 734 * Executor that performs a set-times FileSystemAccess files system operation. 735 */ 736 @InterfaceAudience.Private 737 public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> { 738 private Path path; 739 private long mTime; 740 private long aTime; 741 742 /** 743 * Creates a set-times executor. 744 * 745 * @param path path to set the times. 746 * @param mTime modified time to set. 747 * @param aTime access time to set. 748 */ 749 public FSSetTimes(String path, long mTime, long aTime) { 750 this.path = new Path(path); 751 this.mTime = mTime; 752 this.aTime = aTime; 753 } 754 755 /** 756 * Executes the filesystem operation. 757 * 758 * @param fs filesystem instance to use. 759 * 760 * @return void. 761 * 762 * @throws IOException thrown if an IO error occured. 763 */ 764 @Override 765 public Void execute(FileSystem fs) throws IOException { 766 fs.setTimes(path, mTime, aTime); 767 return null; 768 } 769 770 } 771 772 }