001/**
002* Licensed to the Apache Software Foundation (ASF) under one
003* or more contributor license agreements.  See the NOTICE file
004* distributed with this work for additional information
005* regarding copyright ownership.  The ASF licenses this file
006* to you under the Apache License, Version 2.0 (the
007* "License"); you may not use this file except in compliance
008* with the License.  You may obtain a copy of the License at
009*
010*     http://www.apache.org/licenses/LICENSE-2.0
011*
012* Unless required by applicable law or agreed to in writing, software
013* distributed under the License is distributed on an "AS IS" BASIS,
014* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015* See the License for the specific language governing permissions and
016* limitations under the License.
017*/
018
019package org.apache.hadoop.fs.http.server;
020
021import org.apache.hadoop.fs.ContentSummary;
022import org.apache.hadoop.fs.FileChecksum;
023import org.apache.hadoop.fs.FileStatus;
024import org.apache.hadoop.fs.FileSystem;
025import org.apache.hadoop.fs.GlobFilter;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.fs.PathFilter;
028import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
029import org.apache.hadoop.fs.permission.FsPermission;
030import org.apache.hadoop.io.IOUtils;
031import org.apache.hadoop.lib.service.FileSystemAccess;
032import org.json.simple.JSONArray;
033import org.json.simple.JSONObject;
034
035import java.io.IOException;
036import java.io.InputStream;
037import java.io.OutputStream;
038import java.util.LinkedHashMap;
039import java.util.Map;
040
041/**
042 * FileSystem operation executors used by {@link HttpFSServer}.
043 */
044public class FSOperations {
045
046  /**
047   * Converts a Unix permission octal & symbolic representation
048   * (i.e. 655 or -rwxr--r--) into a FileSystemAccess permission.
049   *
050   * @param str Unix permission symbolic representation.
051   *
052   * @return the FileSystemAccess permission. If the given string was
053   *         'default', it returns <code>FsPermission.getDefault()</code>.
054   */
055  private static FsPermission getPermission(String str) {
056    FsPermission permission;
057    if (str.equals(HttpFSFileSystem.DEFAULT_PERMISSION)) {
058      permission = FsPermission.getDefault();
059    } else if (str.length() == 3) {
060      permission = new FsPermission(Short.parseShort(str, 8));
061    } else {
062      permission = FsPermission.valueOf(str);
063    }
064    return permission;
065  }
066
067  @SuppressWarnings({"unchecked", "deprecation"})
068  private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) {
069    Map json = new LinkedHashMap();
070    json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName());
071    json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString());
072    json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen());
073    json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner());
074    json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup());
075    json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission()));
076    json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime());
077    json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime());
078    json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize());
079    json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication());
080    return json;
081  }
082
083  /**
084   * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON
085   * object.
086   *
087   * @param status FileSystemAccess file status.
088   *
089   * @return The JSON representation of the file status.
090   */
091  @SuppressWarnings({"unchecked", "deprecation"})
092  private static Map fileStatusToJSON(FileStatus status) {
093    Map json = new LinkedHashMap();
094    json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true));
095    return json;
096  }
097
098  /**
099   * Converts a <code>FileChecksum</code> object into a JSON array
100   * object.
101   *
102   * @param checksum file checksum.
103   *
104   * @return The JSON representation of the file checksum.
105   */
106  @SuppressWarnings({"unchecked"})
107  private static Map fileChecksumToJSON(FileChecksum checksum) {
108    Map json = new LinkedHashMap();
109    json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName());
110    json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON,
111             org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes()));
112    json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength());
113    Map response = new LinkedHashMap();
114    response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json);
115    return response;
116  }
117
118  /**
119   * Converts a <code>ContentSummary</code> object into a JSON array
120   * object.
121   *
122   * @param contentSummary the content summary
123   *
124   * @return The JSON representation of the content summary.
125   */
126  @SuppressWarnings({"unchecked"})
127  private static Map contentSummaryToJSON(ContentSummary contentSummary) {
128    Map json = new LinkedHashMap();
129    json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount());
130    json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount());
131    json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength());
132    json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota());
133    json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed());
134    json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota());
135    Map response = new LinkedHashMap();
136    response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json);
137    return response;
138  }
139
140  /**
141   * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array
142   * object.
143   *
144   * @param status FileSystemAccess file status array.
145   * <code>SCHEME://HOST:PORT</code> in the file status.
146   *
147   * @return The JSON representation of the file status array.
148   */
149  @SuppressWarnings("unchecked")
150  private static Map fileStatusToJSON(FileStatus[] status) {
151    JSONArray json = new JSONArray();
152    if (status != null) {
153      for (FileStatus s : status) {
154        json.add(fileStatusToJSONRaw(s, false));
155      }
156    }
157    Map response = new LinkedHashMap();
158    Map temp = new LinkedHashMap();
159    temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json);
160    response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp);
161    return response;
162  }
163
164  /**
165   * Converts an object into a Json Map with with one key-value entry.
166   * <p/>
167   * It assumes the given value is either a JSON primitive type or a
168   * <code>JsonAware</code> instance.
169   *
170   * @param name name for the key of the entry.
171   * @param value for the value of the entry.
172   *
173   * @return the JSON representation of the key-value pair.
174   */
175  @SuppressWarnings("unchecked")
176  private static JSONObject toJSON(String name, Object value) {
177    JSONObject json = new JSONObject();
178    json.put(name, value);
179    return json;
180  }
181
182  /**
183   * Executor that performs an append FileSystemAccess files system operation.
184   */
185  public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> {
186    private InputStream is;
187    private Path path;
188
189    /**
190     * Creates an Append executor.
191     *
192     * @param is input stream to append.
193     * @param path path of the file to append.
194     */
195    public FSAppend(InputStream is, String path) {
196      this.is = is;
197      this.path = new Path(path);
198    }
199
200    /**
201     * Executes the filesystem operation.
202     *
203     * @param fs filesystem instance to use.
204     *
205     * @return void.
206     *
207     * @throws IOException thrown if an IO error occured.
208     */
209    @Override
210    public Void execute(FileSystem fs) throws IOException {
211      int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
212      OutputStream os = fs.append(path, bufferSize);
213      IOUtils.copyBytes(is, os, bufferSize, true);
214      os.close();
215      return null;
216    }
217
218  }
219
220  /**
221   * Executor that performs a content-summary FileSystemAccess files system operation.
222   */
223  public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> {
224    private Path path;
225
226    /**
227     * Creates a content-summary executor.
228     *
229     * @param path the path to retrieve the content-summary.
230     */
231    public FSContentSummary(String path) {
232      this.path = new Path(path);
233    }
234
235    /**
236     * Executes the filesystem operation.
237     *
238     * @param fs filesystem instance to use.
239     *
240     * @return a Map object (JSON friendly) with the content-summary.
241     *
242     * @throws IOException thrown if an IO error occured.
243     */
244    @Override
245    public Map execute(FileSystem fs) throws IOException {
246      ContentSummary contentSummary = fs.getContentSummary(path);
247      return contentSummaryToJSON(contentSummary);
248    }
249
250  }
251
252  /**
253   * Executor that performs a create FileSystemAccess files system operation.
254   */
255  public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> {
256    private InputStream is;
257    private Path path;
258    private String permission;
259    private boolean override;
260    private short replication;
261    private long blockSize;
262
263    /**
264     * Creates a Create executor.
265     *
266     * @param is input stream to for the file to create.
267     * @param path path of the file to create.
268     * @param perm permission for the file.
269     * @param override if the file should be overriden if it already exist.
270     * @param repl the replication factor for the file.
271     * @param blockSize the block size for the file.
272     */
273    public FSCreate(InputStream is, String path, String perm, boolean override, short repl, long blockSize) {
274      this.is = is;
275      this.path = new Path(path);
276      this.permission = perm;
277      this.override = override;
278      this.replication = repl;
279      this.blockSize = blockSize;
280    }
281
282    /**
283     * Executes the filesystem operation.
284     *
285     * @param fs filesystem instance to use.
286     *
287     * @return The URI of the created file.
288     *
289     * @throws IOException thrown if an IO error occured.
290     */
291    @Override
292    public Void execute(FileSystem fs) throws IOException {
293      if (replication == -1) {
294        replication = fs.getDefaultReplication();
295      }
296      if (blockSize == -1) {
297        blockSize = fs.getDefaultBlockSize();
298      }
299      FsPermission fsPermission = getPermission(permission);
300      int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
301      OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
302      IOUtils.copyBytes(is, os, bufferSize, true);
303      os.close();
304      return null;
305    }
306
307  }
308
309  /**
310   * Executor that performs a delete FileSystemAccess files system operation.
311   */
312  public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> {
313    private Path path;
314    private boolean recursive;
315
316    /**
317     * Creates a Delete executor.
318     *
319     * @param path path to delete.
320     * @param recursive if the delete should be recursive or not.
321     */
322    public FSDelete(String path, boolean recursive) {
323      this.path = new Path(path);
324      this.recursive = recursive;
325    }
326
327    /**
328     * Executes the filesystem operation.
329     *
330     * @param fs filesystem instance to use.
331     *
332     * @return <code>true</code> if the delete operation was successful,
333     *         <code>false</code> otherwise.
334     *
335     * @throws IOException thrown if an IO error occured.
336     */
337    @Override
338    public JSONObject execute(FileSystem fs) throws IOException {
339      boolean deleted = fs.delete(path, recursive);
340      return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted);
341    }
342
343  }
344
345  /**
346   * Executor that performs a file-checksum FileSystemAccess files system operation.
347   */
348  public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> {
349    private Path path;
350
351    /**
352     * Creates a file-checksum executor.
353     *
354     * @param path the path to retrieve the checksum.
355     */
356    public FSFileChecksum(String path) {
357      this.path = new Path(path);
358    }
359
360    /**
361     * Executes the filesystem operation.
362     *
363     * @param fs filesystem instance to use.
364     *
365     * @return a Map object (JSON friendly) with the file checksum.
366     *
367     * @throws IOException thrown if an IO error occured.
368     */
369    @Override
370    public Map execute(FileSystem fs) throws IOException {
371      FileChecksum checksum = fs.getFileChecksum(path);
372      return fileChecksumToJSON(checksum);
373    }
374
375  }
376
377  /**
378   * Executor that performs a file-status FileSystemAccess files system operation.
379   */
380  public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> {
381    private Path path;
382
383    /**
384     * Creates a file-status executor.
385     *
386     * @param path the path to retrieve the status.
387     */
388    public FSFileStatus(String path) {
389      this.path = new Path(path);
390    }
391
392    /**
393     * Executes the filesystem operation.
394     *
395     * @param fs filesystem instance to use.
396     *
397     * @return a Map object (JSON friendly) with the file status.
398     *
399     * @throws IOException thrown if an IO error occured.
400     */
401    @Override
402    public Map execute(FileSystem fs) throws IOException {
403      FileStatus status = fs.getFileStatus(path);
404      return fileStatusToJSON(status);
405    }
406
407  }
408
409  /**
410   * Executor that performs a home-dir FileSystemAccess files system operation.
411   */
412  public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> {
413
414    /**
415     * Executes the filesystem operation.
416     *
417     * @param fs filesystem instance to use.
418     *
419     * @return a JSON object with the user home directory.
420     *
421     * @throws IOException thrown if an IO error occured.
422     */
423    @Override
424    @SuppressWarnings("unchecked")
425    public JSONObject execute(FileSystem fs) throws IOException {
426      Path homeDir = fs.getHomeDirectory();
427      JSONObject json = new JSONObject();
428      json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath());
429      return json;
430    }
431
432  }
433
434  /**
435   * Executor that performs a list-status FileSystemAccess files system operation.
436   */
437  public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter {
438    private Path path;
439    private PathFilter filter;
440
441    /**
442     * Creates a list-status executor.
443     *
444     * @param path the directory to retrieve the status of its contents.
445     * @param filter glob filter to use.
446     *
447     * @throws IOException thrown if the filter expression is incorrect.
448     */
449    public FSListStatus(String path, String filter) throws IOException {
450      this.path = new Path(path);
451      this.filter = (filter == null) ? this : new GlobFilter(filter);
452    }
453
454    /**
455     * Executes the filesystem operation.
456     *
457     * @param fs filesystem instance to use.
458     *
459     * @return a Map with the file status of the directory
460     *         contents.
461     *
462     * @throws IOException thrown if an IO error occured.
463     */
464    @Override
465    public Map execute(FileSystem fs) throws IOException {
466      FileStatus[] status = fs.listStatus(path, filter);
467      return fileStatusToJSON(status);
468    }
469
470    @Override
471    public boolean accept(Path path) {
472      return true;
473    }
474
475  }
476
477  /**
478   * Executor that performs a mkdirs FileSystemAccess files system operation.
479   */
480  public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> {
481
482    private Path path;
483    private String permission;
484
485    /**
486     * Creates a mkdirs executor.
487     *
488     * @param path directory path to create.
489     * @param permission permission to use.
490     */
491    public FSMkdirs(String path, String permission) {
492      this.path = new Path(path);
493      this.permission = permission;
494    }
495
496    /**
497     * Executes the filesystem operation.
498     *
499     * @param fs filesystem instance to use.
500     *
501     * @return <code>true</code> if the mkdirs operation was successful,
502     *         <code>false</code> otherwise.
503     *
504     * @throws IOException thrown if an IO error occured.
505     */
506    @Override
507    public JSONObject execute(FileSystem fs) throws IOException {
508      FsPermission fsPermission = getPermission(permission);
509      boolean mkdirs = fs.mkdirs(path, fsPermission);
510      return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs);
511    }
512
513  }
514
515  /**
516   * Executor that performs a open FileSystemAccess files system operation.
517   */
518  public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> {
519    private Path path;
520
521    /**
522     * Creates a open executor.
523     *
524     * @param path file to open.
525     */
526    public FSOpen(String path) {
527      this.path = new Path(path);
528    }
529
530    /**
531     * Executes the filesystem operation.
532     *
533     * @param fs filesystem instance to use.
534     *
535     * @return The inputstream of the file.
536     *
537     * @throws IOException thrown if an IO error occured.
538     */
539    @Override
540    public InputStream execute(FileSystem fs) throws IOException {
541      int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096);
542      return fs.open(path, bufferSize);
543    }
544
545  }
546
547  /**
548   * Executor that performs a rename FileSystemAccess files system operation.
549   */
550  public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> {
551    private Path path;
552    private Path toPath;
553
554    /**
555     * Creates a rename executor.
556     *
557     * @param path path to rename.
558     * @param toPath new name.
559     */
560    public FSRename(String path, String toPath) {
561      this.path = new Path(path);
562      this.toPath = new Path(toPath);
563    }
564
565    /**
566     * Executes the filesystem operation.
567     *
568     * @param fs filesystem instance to use.
569     *
570     * @return <code>true</code> if the rename operation was successful,
571     *         <code>false</code> otherwise.
572     *
573     * @throws IOException thrown if an IO error occured.
574     */
575    @Override
576    public JSONObject execute(FileSystem fs) throws IOException {
577      boolean renamed = fs.rename(path, toPath);
578      return toJSON(HttpFSFileSystem.RENAME_JSON, renamed);
579    }
580
581  }
582
583  /**
584   * Executor that performs a set-owner FileSystemAccess files system operation.
585   */
586  public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> {
587    private Path path;
588    private String owner;
589    private String group;
590
591    /**
592     * Creates a set-owner executor.
593     *
594     * @param path the path to set the owner.
595     * @param owner owner to set.
596     * @param group group to set.
597     */
598    public FSSetOwner(String path, String owner, String group) {
599      this.path = new Path(path);
600      this.owner = owner;
601      this.group = group;
602    }
603
604    /**
605     * Executes the filesystem operation.
606     *
607     * @param fs filesystem instance to use.
608     *
609     * @return void.
610     *
611     * @throws IOException thrown if an IO error occured.
612     */
613    @Override
614    public Void execute(FileSystem fs) throws IOException {
615      fs.setOwner(path, owner, group);
616      return null;
617    }
618
619  }
620
621  /**
622   * Executor that performs a set-permission FileSystemAccess files system operation.
623   */
624  public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> {
625
626    private Path path;
627    private String permission;
628
629    /**
630     * Creates a set-permission executor.
631     *
632     * @param path path to set the permission.
633     * @param permission permission to set.
634     */
635    public FSSetPermission(String path, String permission) {
636      this.path = new Path(path);
637      this.permission = permission;
638    }
639
640    /**
641     * Executes the filesystem operation.
642     *
643     * @param fs filesystem instance to use.
644     *
645     * @return void.
646     *
647     * @throws IOException thrown if an IO error occured.
648     */
649    @Override
650    public Void execute(FileSystem fs) throws IOException {
651      FsPermission fsPermission = getPermission(permission);
652      fs.setPermission(path, fsPermission);
653      return null;
654    }
655
656  }
657
658  /**
659   * Executor that performs a set-replication FileSystemAccess files system operation.
660   */
661  public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> {
662    private Path path;
663    private short replication;
664
665    /**
666     * Creates a set-replication executor.
667     *
668     * @param path path to set the replication factor.
669     * @param replication replication factor to set.
670     */
671    public FSSetReplication(String path, short replication) {
672      this.path = new Path(path);
673      this.replication = replication;
674    }
675
676    /**
677     * Executes the filesystem operation.
678     *
679     * @param fs filesystem instance to use.
680     *
681     * @return <code>true</code> if the replication value was set,
682     *         <code>false</code> otherwise.
683     *
684     * @throws IOException thrown if an IO error occured.
685     */
686    @Override
687    @SuppressWarnings("unchecked")
688    public JSONObject execute(FileSystem fs) throws IOException {
689      boolean ret = fs.setReplication(path, replication);
690      JSONObject json = new JSONObject();
691      json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret);
692      return json;
693    }
694
695  }
696
697  /**
698   * Executor that performs a set-times FileSystemAccess files system operation.
699   */
700  public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> {
701    private Path path;
702    private long mTime;
703    private long aTime;
704
705    /**
706     * Creates a set-times executor.
707     *
708     * @param path path to set the times.
709     * @param mTime modified time to set.
710     * @param aTime access time to set.
711     */
712    public FSSetTimes(String path, long mTime, long aTime) {
713      this.path = new Path(path);
714      this.mTime = mTime;
715      this.aTime = aTime;
716    }
717
718    /**
719     * Executes the filesystem operation.
720     *
721     * @param fs filesystem instance to use.
722     *
723     * @return void.
724     *
725     * @throws IOException thrown if an IO error occured.
726     */
727    @Override
728    public Void execute(FileSystem fs) throws IOException {
729      fs.setTimes(path, mTime, aTime);
730      return null;
731    }
732
733  }
734
735}