001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.fs.http.server; 020 021 import org.apache.hadoop.fs.ContentSummary; 022 import org.apache.hadoop.fs.FileChecksum; 023 import org.apache.hadoop.fs.FileStatus; 024 import org.apache.hadoop.fs.FileSystem; 025 import org.apache.hadoop.fs.GlobFilter; 026 import org.apache.hadoop.fs.Path; 027 import org.apache.hadoop.fs.PathFilter; 028 import org.apache.hadoop.fs.http.client.HttpFSFileSystem; 029 import org.apache.hadoop.fs.permission.FsPermission; 030 import org.apache.hadoop.io.IOUtils; 031 import org.apache.hadoop.lib.service.FileSystemAccess; 032 import org.json.simple.JSONArray; 033 import org.json.simple.JSONObject; 034 035 import java.io.IOException; 036 import java.io.InputStream; 037 import java.io.OutputStream; 038 import java.util.LinkedHashMap; 039 import java.util.Map; 040 041 /** 042 * FileSystem operation executors used by {@link HttpFSServer}. 043 */ 044 public class FSOperations { 045 046 /** 047 * Converts a Unix permission octal & symbolic representation 048 * (i.e. 655 or -rwxr--r--) into a FileSystemAccess permission. 049 * 050 * @param str Unix permission symbolic representation. 051 * 052 * @return the FileSystemAccess permission. If the given string was 053 * 'default', it returns <code>FsPermission.getDefault()</code>. 054 */ 055 private static FsPermission getPermission(String str) { 056 FsPermission permission; 057 if (str.equals(HttpFSFileSystem.DEFAULT_PERMISSION)) { 058 permission = FsPermission.getDefault(); 059 } else if (str.length() == 3) { 060 permission = new FsPermission(Short.parseShort(str, 8)); 061 } else { 062 permission = FsPermission.valueOf(str); 063 } 064 return permission; 065 } 066 067 @SuppressWarnings({"unchecked", "deprecation"}) 068 private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) { 069 Map json = new LinkedHashMap(); 070 json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName()); 071 json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString()); 072 json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen()); 073 json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner()); 074 json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup()); 075 json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission())); 076 json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime()); 077 json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime()); 078 json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize()); 079 json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication()); 080 return json; 081 } 082 083 /** 084 * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON 085 * object. 086 * 087 * @param status FileSystemAccess file status. 088 * 089 * @return The JSON representation of the file status. 090 */ 091 @SuppressWarnings({"unchecked", "deprecation"}) 092 private static Map fileStatusToJSON(FileStatus status) { 093 Map json = new LinkedHashMap(); 094 json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true)); 095 return json; 096 } 097 098 /** 099 * Converts a <code>FileChecksum</code> object into a JSON array 100 * object. 101 * 102 * @param checksum file checksum. 103 * 104 * @return The JSON representation of the file checksum. 105 */ 106 @SuppressWarnings({"unchecked"}) 107 private static Map fileChecksumToJSON(FileChecksum checksum) { 108 Map json = new LinkedHashMap(); 109 json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName()); 110 json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON, 111 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes())); 112 json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength()); 113 Map response = new LinkedHashMap(); 114 response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json); 115 return response; 116 } 117 118 /** 119 * Converts a <code>ContentSummary</code> object into a JSON array 120 * object. 121 * 122 * @param contentSummary the content summary 123 * 124 * @return The JSON representation of the content summary. 125 */ 126 @SuppressWarnings({"unchecked"}) 127 private static Map contentSummaryToJSON(ContentSummary contentSummary) { 128 Map json = new LinkedHashMap(); 129 json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount()); 130 json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount()); 131 json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength()); 132 json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota()); 133 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed()); 134 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota()); 135 Map response = new LinkedHashMap(); 136 response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json); 137 return response; 138 } 139 140 /** 141 * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array 142 * object. 143 * 144 * @param status FileSystemAccess file status array. 145 * <code>SCHEME://HOST:PORT</code> in the file status. 146 * 147 * @return The JSON representation of the file status array. 148 */ 149 @SuppressWarnings("unchecked") 150 private static Map fileStatusToJSON(FileStatus[] status) { 151 JSONArray json = new JSONArray(); 152 if (status != null) { 153 for (FileStatus s : status) { 154 json.add(fileStatusToJSONRaw(s, false)); 155 } 156 } 157 Map response = new LinkedHashMap(); 158 Map temp = new LinkedHashMap(); 159 temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json); 160 response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp); 161 return response; 162 } 163 164 /** 165 * Converts an object into a Json Map with with one key-value entry. 166 * <p/> 167 * It assumes the given value is either a JSON primitive type or a 168 * <code>JsonAware</code> instance. 169 * 170 * @param name name for the key of the entry. 171 * @param value for the value of the entry. 172 * 173 * @return the JSON representation of the key-value pair. 174 */ 175 @SuppressWarnings("unchecked") 176 private static JSONObject toJSON(String name, Object value) { 177 JSONObject json = new JSONObject(); 178 json.put(name, value); 179 return json; 180 } 181 182 /** 183 * Executor that performs an append FileSystemAccess files system operation. 184 */ 185 public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> { 186 private InputStream is; 187 private Path path; 188 189 /** 190 * Creates an Append executor. 191 * 192 * @param is input stream to append. 193 * @param path path of the file to append. 194 */ 195 public FSAppend(InputStream is, String path) { 196 this.is = is; 197 this.path = new Path(path); 198 } 199 200 /** 201 * Executes the filesystem operation. 202 * 203 * @param fs filesystem instance to use. 204 * 205 * @return void. 206 * 207 * @throws IOException thrown if an IO error occured. 208 */ 209 @Override 210 public Void execute(FileSystem fs) throws IOException { 211 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 212 OutputStream os = fs.append(path, bufferSize); 213 IOUtils.copyBytes(is, os, bufferSize, true); 214 os.close(); 215 return null; 216 } 217 218 } 219 220 /** 221 * Executor that performs a content-summary FileSystemAccess files system operation. 222 */ 223 public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> { 224 private Path path; 225 226 /** 227 * Creates a content-summary executor. 228 * 229 * @param path the path to retrieve the content-summary. 230 */ 231 public FSContentSummary(String path) { 232 this.path = new Path(path); 233 } 234 235 /** 236 * Executes the filesystem operation. 237 * 238 * @param fs filesystem instance to use. 239 * 240 * @return a Map object (JSON friendly) with the content-summary. 241 * 242 * @throws IOException thrown if an IO error occured. 243 */ 244 @Override 245 public Map execute(FileSystem fs) throws IOException { 246 ContentSummary contentSummary = fs.getContentSummary(path); 247 return contentSummaryToJSON(contentSummary); 248 } 249 250 } 251 252 /** 253 * Executor that performs a create FileSystemAccess files system operation. 254 */ 255 public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> { 256 private InputStream is; 257 private Path path; 258 private String permission; 259 private boolean override; 260 private short replication; 261 private long blockSize; 262 263 /** 264 * Creates a Create executor. 265 * 266 * @param is input stream to for the file to create. 267 * @param path path of the file to create. 268 * @param perm permission for the file. 269 * @param override if the file should be overriden if it already exist. 270 * @param repl the replication factor for the file. 271 * @param blockSize the block size for the file. 272 */ 273 public FSCreate(InputStream is, String path, String perm, boolean override, short repl, long blockSize) { 274 this.is = is; 275 this.path = new Path(path); 276 this.permission = perm; 277 this.override = override; 278 this.replication = repl; 279 this.blockSize = blockSize; 280 } 281 282 /** 283 * Executes the filesystem operation. 284 * 285 * @param fs filesystem instance to use. 286 * 287 * @return The URI of the created file. 288 * 289 * @throws IOException thrown if an IO error occured. 290 */ 291 @Override 292 public Void execute(FileSystem fs) throws IOException { 293 if (replication == -1) { 294 replication = fs.getDefaultReplication(); 295 } 296 if (blockSize == -1) { 297 blockSize = fs.getDefaultBlockSize(); 298 } 299 FsPermission fsPermission = getPermission(permission); 300 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 301 OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null); 302 IOUtils.copyBytes(is, os, bufferSize, true); 303 os.close(); 304 return null; 305 } 306 307 } 308 309 /** 310 * Executor that performs a delete FileSystemAccess files system operation. 311 */ 312 public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> { 313 private Path path; 314 private boolean recursive; 315 316 /** 317 * Creates a Delete executor. 318 * 319 * @param path path to delete. 320 * @param recursive if the delete should be recursive or not. 321 */ 322 public FSDelete(String path, boolean recursive) { 323 this.path = new Path(path); 324 this.recursive = recursive; 325 } 326 327 /** 328 * Executes the filesystem operation. 329 * 330 * @param fs filesystem instance to use. 331 * 332 * @return <code>true</code> if the delete operation was successful, 333 * <code>false</code> otherwise. 334 * 335 * @throws IOException thrown if an IO error occured. 336 */ 337 @Override 338 public JSONObject execute(FileSystem fs) throws IOException { 339 boolean deleted = fs.delete(path, recursive); 340 return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted); 341 } 342 343 } 344 345 /** 346 * Executor that performs a file-checksum FileSystemAccess files system operation. 347 */ 348 public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> { 349 private Path path; 350 351 /** 352 * Creates a file-checksum executor. 353 * 354 * @param path the path to retrieve the checksum. 355 */ 356 public FSFileChecksum(String path) { 357 this.path = new Path(path); 358 } 359 360 /** 361 * Executes the filesystem operation. 362 * 363 * @param fs filesystem instance to use. 364 * 365 * @return a Map object (JSON friendly) with the file checksum. 366 * 367 * @throws IOException thrown if an IO error occured. 368 */ 369 @Override 370 public Map execute(FileSystem fs) throws IOException { 371 FileChecksum checksum = fs.getFileChecksum(path); 372 return fileChecksumToJSON(checksum); 373 } 374 375 } 376 377 /** 378 * Executor that performs a file-status FileSystemAccess files system operation. 379 */ 380 public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> { 381 private Path path; 382 383 /** 384 * Creates a file-status executor. 385 * 386 * @param path the path to retrieve the status. 387 */ 388 public FSFileStatus(String path) { 389 this.path = new Path(path); 390 } 391 392 /** 393 * Executes the filesystem operation. 394 * 395 * @param fs filesystem instance to use. 396 * 397 * @return a Map object (JSON friendly) with the file status. 398 * 399 * @throws IOException thrown if an IO error occured. 400 */ 401 @Override 402 public Map execute(FileSystem fs) throws IOException { 403 FileStatus status = fs.getFileStatus(path); 404 return fileStatusToJSON(status); 405 } 406 407 } 408 409 /** 410 * Executor that performs a home-dir FileSystemAccess files system operation. 411 */ 412 public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> { 413 414 /** 415 * Executes the filesystem operation. 416 * 417 * @param fs filesystem instance to use. 418 * 419 * @return a JSON object with the user home directory. 420 * 421 * @throws IOException thrown if an IO error occured. 422 */ 423 @Override 424 @SuppressWarnings("unchecked") 425 public JSONObject execute(FileSystem fs) throws IOException { 426 Path homeDir = fs.getHomeDirectory(); 427 JSONObject json = new JSONObject(); 428 json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath()); 429 return json; 430 } 431 432 } 433 434 /** 435 * Executor that performs a list-status FileSystemAccess files system operation. 436 */ 437 public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter { 438 private Path path; 439 private PathFilter filter; 440 441 /** 442 * Creates a list-status executor. 443 * 444 * @param path the directory to retrieve the status of its contents. 445 * @param filter glob filter to use. 446 * 447 * @throws IOException thrown if the filter expression is incorrect. 448 */ 449 public FSListStatus(String path, String filter) throws IOException { 450 this.path = new Path(path); 451 this.filter = (filter == null) ? this : new GlobFilter(filter); 452 } 453 454 /** 455 * Executes the filesystem operation. 456 * 457 * @param fs filesystem instance to use. 458 * 459 * @return a Map with the file status of the directory 460 * contents. 461 * 462 * @throws IOException thrown if an IO error occured. 463 */ 464 @Override 465 public Map execute(FileSystem fs) throws IOException { 466 FileStatus[] status = fs.listStatus(path, filter); 467 return fileStatusToJSON(status); 468 } 469 470 @Override 471 public boolean accept(Path path) { 472 return true; 473 } 474 475 } 476 477 /** 478 * Executor that performs a mkdirs FileSystemAccess files system operation. 479 */ 480 public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> { 481 482 private Path path; 483 private String permission; 484 485 /** 486 * Creates a mkdirs executor. 487 * 488 * @param path directory path to create. 489 * @param permission permission to use. 490 */ 491 public FSMkdirs(String path, String permission) { 492 this.path = new Path(path); 493 this.permission = permission; 494 } 495 496 /** 497 * Executes the filesystem operation. 498 * 499 * @param fs filesystem instance to use. 500 * 501 * @return <code>true</code> if the mkdirs operation was successful, 502 * <code>false</code> otherwise. 503 * 504 * @throws IOException thrown if an IO error occured. 505 */ 506 @Override 507 public JSONObject execute(FileSystem fs) throws IOException { 508 FsPermission fsPermission = getPermission(permission); 509 boolean mkdirs = fs.mkdirs(path, fsPermission); 510 return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs); 511 } 512 513 } 514 515 /** 516 * Executor that performs a open FileSystemAccess files system operation. 517 */ 518 public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> { 519 private Path path; 520 521 /** 522 * Creates a open executor. 523 * 524 * @param path file to open. 525 */ 526 public FSOpen(String path) { 527 this.path = new Path(path); 528 } 529 530 /** 531 * Executes the filesystem operation. 532 * 533 * @param fs filesystem instance to use. 534 * 535 * @return The inputstream of the file. 536 * 537 * @throws IOException thrown if an IO error occured. 538 */ 539 @Override 540 public InputStream execute(FileSystem fs) throws IOException { 541 int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096); 542 return fs.open(path, bufferSize); 543 } 544 545 } 546 547 /** 548 * Executor that performs a rename FileSystemAccess files system operation. 549 */ 550 public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> { 551 private Path path; 552 private Path toPath; 553 554 /** 555 * Creates a rename executor. 556 * 557 * @param path path to rename. 558 * @param toPath new name. 559 */ 560 public FSRename(String path, String toPath) { 561 this.path = new Path(path); 562 this.toPath = new Path(toPath); 563 } 564 565 /** 566 * Executes the filesystem operation. 567 * 568 * @param fs filesystem instance to use. 569 * 570 * @return <code>true</code> if the rename operation was successful, 571 * <code>false</code> otherwise. 572 * 573 * @throws IOException thrown if an IO error occured. 574 */ 575 @Override 576 public JSONObject execute(FileSystem fs) throws IOException { 577 boolean renamed = fs.rename(path, toPath); 578 return toJSON(HttpFSFileSystem.RENAME_JSON, renamed); 579 } 580 581 } 582 583 /** 584 * Executor that performs a set-owner FileSystemAccess files system operation. 585 */ 586 public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> { 587 private Path path; 588 private String owner; 589 private String group; 590 591 /** 592 * Creates a set-owner executor. 593 * 594 * @param path the path to set the owner. 595 * @param owner owner to set. 596 * @param group group to set. 597 */ 598 public FSSetOwner(String path, String owner, String group) { 599 this.path = new Path(path); 600 this.owner = owner; 601 this.group = group; 602 } 603 604 /** 605 * Executes the filesystem operation. 606 * 607 * @param fs filesystem instance to use. 608 * 609 * @return void. 610 * 611 * @throws IOException thrown if an IO error occured. 612 */ 613 @Override 614 public Void execute(FileSystem fs) throws IOException { 615 fs.setOwner(path, owner, group); 616 return null; 617 } 618 619 } 620 621 /** 622 * Executor that performs a set-permission FileSystemAccess files system operation. 623 */ 624 public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> { 625 626 private Path path; 627 private String permission; 628 629 /** 630 * Creates a set-permission executor. 631 * 632 * @param path path to set the permission. 633 * @param permission permission to set. 634 */ 635 public FSSetPermission(String path, String permission) { 636 this.path = new Path(path); 637 this.permission = permission; 638 } 639 640 /** 641 * Executes the filesystem operation. 642 * 643 * @param fs filesystem instance to use. 644 * 645 * @return void. 646 * 647 * @throws IOException thrown if an IO error occured. 648 */ 649 @Override 650 public Void execute(FileSystem fs) throws IOException { 651 FsPermission fsPermission = getPermission(permission); 652 fs.setPermission(path, fsPermission); 653 return null; 654 } 655 656 } 657 658 /** 659 * Executor that performs a set-replication FileSystemAccess files system operation. 660 */ 661 public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> { 662 private Path path; 663 private short replication; 664 665 /** 666 * Creates a set-replication executor. 667 * 668 * @param path path to set the replication factor. 669 * @param replication replication factor to set. 670 */ 671 public FSSetReplication(String path, short replication) { 672 this.path = new Path(path); 673 this.replication = replication; 674 } 675 676 /** 677 * Executes the filesystem operation. 678 * 679 * @param fs filesystem instance to use. 680 * 681 * @return <code>true</code> if the replication value was set, 682 * <code>false</code> otherwise. 683 * 684 * @throws IOException thrown if an IO error occured. 685 */ 686 @Override 687 @SuppressWarnings("unchecked") 688 public JSONObject execute(FileSystem fs) throws IOException { 689 boolean ret = fs.setReplication(path, replication); 690 JSONObject json = new JSONObject(); 691 json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret); 692 return json; 693 } 694 695 } 696 697 /** 698 * Executor that performs a set-times FileSystemAccess files system operation. 699 */ 700 public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> { 701 private Path path; 702 private long mTime; 703 private long aTime; 704 705 /** 706 * Creates a set-times executor. 707 * 708 * @param path path to set the times. 709 * @param mTime modified time to set. 710 * @param aTime access time to set. 711 */ 712 public FSSetTimes(String path, long mTime, long aTime) { 713 this.path = new Path(path); 714 this.mTime = mTime; 715 this.aTime = aTime; 716 } 717 718 /** 719 * Executes the filesystem operation. 720 * 721 * @param fs filesystem instance to use. 722 * 723 * @return void. 724 * 725 * @throws IOException thrown if an IO error occured. 726 */ 727 @Override 728 public Void execute(FileSystem fs) throws IOException { 729 fs.setTimes(path, mTime, aTime); 730 return null; 731 } 732 733 } 734 735 }