001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs.http.server; 019 020 import org.apache.hadoop.fs.ContentSummary; 021 import org.apache.hadoop.fs.FileChecksum; 022 import org.apache.hadoop.fs.FileStatus; 023 import org.apache.hadoop.fs.FileSystem; 024 import org.apache.hadoop.fs.GlobFilter; 025 import org.apache.hadoop.fs.Path; 026 import org.apache.hadoop.fs.PathFilter; 027 import org.apache.hadoop.fs.http.client.HttpFSFileSystem; 028 import org.apache.hadoop.fs.permission.FsPermission; 029 import org.apache.hadoop.io.IOUtils; 030 import org.apache.hadoop.lib.service.FileSystemAccess; 031 import org.json.simple.JSONArray; 032 import org.json.simple.JSONObject; 033 034 import java.io.IOException; 035 import java.io.InputStream; 036 import java.io.OutputStream; 037 import java.util.LinkedHashMap; 038 import java.util.Map; 039 040 /** 041 * FileSystem operation executors used by {@link HttpFSServer}. 042 */ 043 public class FSOperations { 044 045 /** 046 * Converts a Unix permission octal 047 * (i.e. 655 or 1777) into a FileSystemAccess permission. 048 * 049 * @param str Unix permission symbolic representation. 050 * 051 * @return the FileSystemAccess permission. If the given string was 052 * 'default', it returns <code>FsPermission.getDefault()</code>. 053 */ 054 private static FsPermission getPermission(String str) { 055 FsPermission permission; 056 if (str.equals(HttpFSFileSystem.DEFAULT_PERMISSION)) { 057 permission = FsPermission.getDefault(); 058 } else { 059 permission = new FsPermission(Short.parseShort(str, 8)); 060 } 061 return permission; 062 } 063 064 @SuppressWarnings({"unchecked", "deprecation"}) 065 private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) { 066 Map json = new LinkedHashMap(); 067 json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName()); 068 json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString()); 069 json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen()); 070 json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner()); 071 json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup()); 072 json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission())); 073 json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime()); 074 json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime()); 075 json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize()); 076 json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication()); 077 return json; 078 } 079 080 /** 081 * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON 082 * object. 083 * 084 * @param status FileSystemAccess file status. 085 * 086 * @return The JSON representation of the file status. 087 */ 088 @SuppressWarnings({"unchecked", "deprecation"}) 089 private static Map fileStatusToJSON(FileStatus status) { 090 Map json = new LinkedHashMap(); 091 json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true)); 092 return json; 093 } 094 095 /** 096 * Converts a <code>FileChecksum</code> object into a JSON array 097 * object. 098 * 099 * @param checksum file checksum. 100 * 101 * @return The JSON representation of the file checksum. 102 */ 103 @SuppressWarnings({"unchecked"}) 104 private static Map fileChecksumToJSON(FileChecksum checksum) { 105 Map json = new LinkedHashMap(); 106 json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName()); 107 json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON, 108 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes())); 109 json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength()); 110 Map response = new LinkedHashMap(); 111 response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json); 112 return response; 113 } 114 115 /** 116 * Converts a <code>ContentSummary</code> object into a JSON array 117 * object. 118 * 119 * @param contentSummary the content summary 120 * 121 * @return The JSON representation of the content summary. 122 */ 123 @SuppressWarnings({"unchecked"}) 124 private static Map contentSummaryToJSON(ContentSummary contentSummary) { 125 Map json = new LinkedHashMap(); 126 json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount()); 127 json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount()); 128 json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength()); 129 json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota()); 130 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed()); 131 json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota()); 132 Map response = new LinkedHashMap(); 133 response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json); 134 return response; 135 } 136 137 /** 138 * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array 139 * object. 140 * 141 * @param status FileSystemAccess file status array. 142 * <code>SCHEME://HOST:PORT</code> in the file status. 143 * 144 * @return The JSON representation of the file status array. 145 */ 146 @SuppressWarnings("unchecked") 147 private static Map fileStatusToJSON(FileStatus[] status) { 148 JSONArray json = new JSONArray(); 149 if (status != null) { 150 for (FileStatus s : status) { 151 json.add(fileStatusToJSONRaw(s, false)); 152 } 153 } 154 Map response = new LinkedHashMap(); 155 Map temp = new LinkedHashMap(); 156 temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json); 157 response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp); 158 return response; 159 } 160 161 /** 162 * Converts an object into a Json Map with with one key-value entry. 163 * <p/> 164 * It assumes the given value is either a JSON primitive type or a 165 * <code>JsonAware</code> instance. 166 * 167 * @param name name for the key of the entry. 168 * @param value for the value of the entry. 169 * 170 * @return the JSON representation of the key-value pair. 171 */ 172 @SuppressWarnings("unchecked") 173 private static JSONObject toJSON(String name, Object value) { 174 JSONObject json = new JSONObject(); 175 json.put(name, value); 176 return json; 177 } 178 179 /** 180 * Executor that performs an append FileSystemAccess files system operation. 181 */ 182 public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> { 183 private InputStream is; 184 private Path path; 185 186 /** 187 * Creates an Append executor. 188 * 189 * @param is input stream to append. 190 * @param path path of the file to append. 191 */ 192 public FSAppend(InputStream is, String path) { 193 this.is = is; 194 this.path = new Path(path); 195 } 196 197 /** 198 * Executes the filesystem operation. 199 * 200 * @param fs filesystem instance to use. 201 * 202 * @return void. 203 * 204 * @throws IOException thrown if an IO error occured. 205 */ 206 @Override 207 public Void execute(FileSystem fs) throws IOException { 208 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 209 OutputStream os = fs.append(path, bufferSize); 210 IOUtils.copyBytes(is, os, bufferSize, true); 211 os.close(); 212 return null; 213 } 214 215 } 216 217 /** 218 * Executor that performs a content-summary FileSystemAccess files system operation. 219 */ 220 public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> { 221 private Path path; 222 223 /** 224 * Creates a content-summary executor. 225 * 226 * @param path the path to retrieve the content-summary. 227 */ 228 public FSContentSummary(String path) { 229 this.path = new Path(path); 230 } 231 232 /** 233 * Executes the filesystem operation. 234 * 235 * @param fs filesystem instance to use. 236 * 237 * @return a Map object (JSON friendly) with the content-summary. 238 * 239 * @throws IOException thrown if an IO error occured. 240 */ 241 @Override 242 public Map execute(FileSystem fs) throws IOException { 243 ContentSummary contentSummary = fs.getContentSummary(path); 244 return contentSummaryToJSON(contentSummary); 245 } 246 247 } 248 249 /** 250 * Executor that performs a create FileSystemAccess files system operation. 251 */ 252 public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> { 253 private InputStream is; 254 private Path path; 255 private String permission; 256 private boolean override; 257 private short replication; 258 private long blockSize; 259 260 /** 261 * Creates a Create executor. 262 * 263 * @param is input stream to for the file to create. 264 * @param path path of the file to create. 265 * @param perm permission for the file. 266 * @param override if the file should be overriden if it already exist. 267 * @param repl the replication factor for the file. 268 * @param blockSize the block size for the file. 269 */ 270 public FSCreate(InputStream is, String path, String perm, boolean override, short repl, long blockSize) { 271 this.is = is; 272 this.path = new Path(path); 273 this.permission = perm; 274 this.override = override; 275 this.replication = repl; 276 this.blockSize = blockSize; 277 } 278 279 /** 280 * Executes the filesystem operation. 281 * 282 * @param fs filesystem instance to use. 283 * 284 * @return The URI of the created file. 285 * 286 * @throws IOException thrown if an IO error occured. 287 */ 288 @Override 289 public Void execute(FileSystem fs) throws IOException { 290 if (replication == -1) { 291 replication = fs.getDefaultReplication(); 292 } 293 if (blockSize == -1) { 294 blockSize = fs.getDefaultBlockSize(); 295 } 296 FsPermission fsPermission = getPermission(permission); 297 int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); 298 OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null); 299 IOUtils.copyBytes(is, os, bufferSize, true); 300 os.close(); 301 return null; 302 } 303 304 } 305 306 /** 307 * Executor that performs a delete FileSystemAccess files system operation. 308 */ 309 public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> { 310 private Path path; 311 private boolean recursive; 312 313 /** 314 * Creates a Delete executor. 315 * 316 * @param path path to delete. 317 * @param recursive if the delete should be recursive or not. 318 */ 319 public FSDelete(String path, boolean recursive) { 320 this.path = new Path(path); 321 this.recursive = recursive; 322 } 323 324 /** 325 * Executes the filesystem operation. 326 * 327 * @param fs filesystem instance to use. 328 * 329 * @return <code>true</code> if the delete operation was successful, 330 * <code>false</code> otherwise. 331 * 332 * @throws IOException thrown if an IO error occured. 333 */ 334 @Override 335 public JSONObject execute(FileSystem fs) throws IOException { 336 boolean deleted = fs.delete(path, recursive); 337 return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted); 338 } 339 340 } 341 342 /** 343 * Executor that performs a file-checksum FileSystemAccess files system operation. 344 */ 345 public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> { 346 private Path path; 347 348 /** 349 * Creates a file-checksum executor. 350 * 351 * @param path the path to retrieve the checksum. 352 */ 353 public FSFileChecksum(String path) { 354 this.path = new Path(path); 355 } 356 357 /** 358 * Executes the filesystem operation. 359 * 360 * @param fs filesystem instance to use. 361 * 362 * @return a Map object (JSON friendly) with the file checksum. 363 * 364 * @throws IOException thrown if an IO error occured. 365 */ 366 @Override 367 public Map execute(FileSystem fs) throws IOException { 368 FileChecksum checksum = fs.getFileChecksum(path); 369 return fileChecksumToJSON(checksum); 370 } 371 372 } 373 374 /** 375 * Executor that performs a file-status FileSystemAccess files system operation. 376 */ 377 public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> { 378 private Path path; 379 380 /** 381 * Creates a file-status executor. 382 * 383 * @param path the path to retrieve the status. 384 */ 385 public FSFileStatus(String path) { 386 this.path = new Path(path); 387 } 388 389 /** 390 * Executes the filesystem operation. 391 * 392 * @param fs filesystem instance to use. 393 * 394 * @return a Map object (JSON friendly) with the file status. 395 * 396 * @throws IOException thrown if an IO error occured. 397 */ 398 @Override 399 public Map execute(FileSystem fs) throws IOException { 400 FileStatus status = fs.getFileStatus(path); 401 return fileStatusToJSON(status); 402 } 403 404 } 405 406 /** 407 * Executor that performs a home-dir FileSystemAccess files system operation. 408 */ 409 public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> { 410 411 /** 412 * Executes the filesystem operation. 413 * 414 * @param fs filesystem instance to use. 415 * 416 * @return a JSON object with the user home directory. 417 * 418 * @throws IOException thrown if an IO error occured. 419 */ 420 @Override 421 @SuppressWarnings("unchecked") 422 public JSONObject execute(FileSystem fs) throws IOException { 423 Path homeDir = fs.getHomeDirectory(); 424 JSONObject json = new JSONObject(); 425 json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath()); 426 return json; 427 } 428 429 } 430 431 /** 432 * Executor that performs a list-status FileSystemAccess files system operation. 433 */ 434 public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter { 435 private Path path; 436 private PathFilter filter; 437 438 /** 439 * Creates a list-status executor. 440 * 441 * @param path the directory to retrieve the status of its contents. 442 * @param filter glob filter to use. 443 * 444 * @throws IOException thrown if the filter expression is incorrect. 445 */ 446 public FSListStatus(String path, String filter) throws IOException { 447 this.path = new Path(path); 448 this.filter = (filter == null) ? this : new GlobFilter(filter); 449 } 450 451 /** 452 * Executes the filesystem operation. 453 * 454 * @param fs filesystem instance to use. 455 * 456 * @return a Map with the file status of the directory 457 * contents. 458 * 459 * @throws IOException thrown if an IO error occured. 460 */ 461 @Override 462 public Map execute(FileSystem fs) throws IOException { 463 FileStatus[] status = fs.listStatus(path, filter); 464 return fileStatusToJSON(status); 465 } 466 467 @Override 468 public boolean accept(Path path) { 469 return true; 470 } 471 472 } 473 474 /** 475 * Executor that performs a mkdirs FileSystemAccess files system operation. 476 */ 477 public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> { 478 479 private Path path; 480 private String permission; 481 482 /** 483 * Creates a mkdirs executor. 484 * 485 * @param path directory path to create. 486 * @param permission permission to use. 487 */ 488 public FSMkdirs(String path, String permission) { 489 this.path = new Path(path); 490 this.permission = permission; 491 } 492 493 /** 494 * Executes the filesystem operation. 495 * 496 * @param fs filesystem instance to use. 497 * 498 * @return <code>true</code> if the mkdirs operation was successful, 499 * <code>false</code> otherwise. 500 * 501 * @throws IOException thrown if an IO error occured. 502 */ 503 @Override 504 public JSONObject execute(FileSystem fs) throws IOException { 505 FsPermission fsPermission = getPermission(permission); 506 boolean mkdirs = fs.mkdirs(path, fsPermission); 507 return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs); 508 } 509 510 } 511 512 /** 513 * Executor that performs a open FileSystemAccess files system operation. 514 */ 515 public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> { 516 private Path path; 517 518 /** 519 * Creates a open executor. 520 * 521 * @param path file to open. 522 */ 523 public FSOpen(String path) { 524 this.path = new Path(path); 525 } 526 527 /** 528 * Executes the filesystem operation. 529 * 530 * @param fs filesystem instance to use. 531 * 532 * @return The inputstream of the file. 533 * 534 * @throws IOException thrown if an IO error occured. 535 */ 536 @Override 537 public InputStream execute(FileSystem fs) throws IOException { 538 int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096); 539 return fs.open(path, bufferSize); 540 } 541 542 } 543 544 /** 545 * Executor that performs a rename FileSystemAccess files system operation. 546 */ 547 public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> { 548 private Path path; 549 private Path toPath; 550 551 /** 552 * Creates a rename executor. 553 * 554 * @param path path to rename. 555 * @param toPath new name. 556 */ 557 public FSRename(String path, String toPath) { 558 this.path = new Path(path); 559 this.toPath = new Path(toPath); 560 } 561 562 /** 563 * Executes the filesystem operation. 564 * 565 * @param fs filesystem instance to use. 566 * 567 * @return <code>true</code> if the rename operation was successful, 568 * <code>false</code> otherwise. 569 * 570 * @throws IOException thrown if an IO error occured. 571 */ 572 @Override 573 public JSONObject execute(FileSystem fs) throws IOException { 574 boolean renamed = fs.rename(path, toPath); 575 return toJSON(HttpFSFileSystem.RENAME_JSON, renamed); 576 } 577 578 } 579 580 /** 581 * Executor that performs a set-owner FileSystemAccess files system operation. 582 */ 583 public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> { 584 private Path path; 585 private String owner; 586 private String group; 587 588 /** 589 * Creates a set-owner executor. 590 * 591 * @param path the path to set the owner. 592 * @param owner owner to set. 593 * @param group group to set. 594 */ 595 public FSSetOwner(String path, String owner, String group) { 596 this.path = new Path(path); 597 this.owner = owner; 598 this.group = group; 599 } 600 601 /** 602 * Executes the filesystem operation. 603 * 604 * @param fs filesystem instance to use. 605 * 606 * @return void. 607 * 608 * @throws IOException thrown if an IO error occured. 609 */ 610 @Override 611 public Void execute(FileSystem fs) throws IOException { 612 fs.setOwner(path, owner, group); 613 return null; 614 } 615 616 } 617 618 /** 619 * Executor that performs a set-permission FileSystemAccess files system operation. 620 */ 621 public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> { 622 623 private Path path; 624 private String permission; 625 626 /** 627 * Creates a set-permission executor. 628 * 629 * @param path path to set the permission. 630 * @param permission permission to set. 631 */ 632 public FSSetPermission(String path, String permission) { 633 this.path = new Path(path); 634 this.permission = permission; 635 } 636 637 /** 638 * Executes the filesystem operation. 639 * 640 * @param fs filesystem instance to use. 641 * 642 * @return void. 643 * 644 * @throws IOException thrown if an IO error occured. 645 */ 646 @Override 647 public Void execute(FileSystem fs) throws IOException { 648 FsPermission fsPermission = getPermission(permission); 649 fs.setPermission(path, fsPermission); 650 return null; 651 } 652 653 } 654 655 /** 656 * Executor that performs a set-replication FileSystemAccess files system operation. 657 */ 658 public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> { 659 private Path path; 660 private short replication; 661 662 /** 663 * Creates a set-replication executor. 664 * 665 * @param path path to set the replication factor. 666 * @param replication replication factor to set. 667 */ 668 public FSSetReplication(String path, short replication) { 669 this.path = new Path(path); 670 this.replication = replication; 671 } 672 673 /** 674 * Executes the filesystem operation. 675 * 676 * @param fs filesystem instance to use. 677 * 678 * @return <code>true</code> if the replication value was set, 679 * <code>false</code> otherwise. 680 * 681 * @throws IOException thrown if an IO error occured. 682 */ 683 @Override 684 @SuppressWarnings("unchecked") 685 public JSONObject execute(FileSystem fs) throws IOException { 686 boolean ret = fs.setReplication(path, replication); 687 JSONObject json = new JSONObject(); 688 json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret); 689 return json; 690 } 691 692 } 693 694 /** 695 * Executor that performs a set-times FileSystemAccess files system operation. 696 */ 697 public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> { 698 private Path path; 699 private long mTime; 700 private long aTime; 701 702 /** 703 * Creates a set-times executor. 704 * 705 * @param path path to set the times. 706 * @param mTime modified time to set. 707 * @param aTime access time to set. 708 */ 709 public FSSetTimes(String path, long mTime, long aTime) { 710 this.path = new Path(path); 711 this.mTime = mTime; 712 this.aTime = aTime; 713 } 714 715 /** 716 * Executes the filesystem operation. 717 * 718 * @param fs filesystem instance to use. 719 * 720 * @return void. 721 * 722 * @throws IOException thrown if an IO error occured. 723 */ 724 @Override 725 public Void execute(FileSystem fs) throws IOException { 726 fs.setTimes(path, mTime, aTime); 727 return null; 728 } 729 730 } 731 732 }