001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs.http.server;
019    
020    import org.apache.hadoop.fs.ContentSummary;
021    import org.apache.hadoop.fs.FileChecksum;
022    import org.apache.hadoop.fs.FileStatus;
023    import org.apache.hadoop.fs.FileSystem;
024    import org.apache.hadoop.fs.GlobFilter;
025    import org.apache.hadoop.fs.Path;
026    import org.apache.hadoop.fs.PathFilter;
027    import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
028    import org.apache.hadoop.fs.permission.FsPermission;
029    import org.apache.hadoop.io.IOUtils;
030    import org.apache.hadoop.lib.service.FileSystemAccess;
031    import org.json.simple.JSONArray;
032    import org.json.simple.JSONObject;
033    
034    import java.io.IOException;
035    import java.io.InputStream;
036    import java.io.OutputStream;
037    import java.util.LinkedHashMap;
038    import java.util.Map;
039    
040    /**
041     * FileSystem operation executors used by {@link HttpFSServer}.
042     */
043    public class FSOperations {
044    
045      /**
046       * Converts a Unix permission octal
047       * (i.e. 655 or 1777) into a FileSystemAccess permission.
048       *
049       * @param str Unix permission symbolic representation.
050       *
051       * @return the FileSystemAccess permission. If the given string was
052       *         'default', it returns <code>FsPermission.getDefault()</code>.
053       */
054      private static FsPermission getPermission(String str) {
055        FsPermission permission;
056        if (str.equals(HttpFSFileSystem.DEFAULT_PERMISSION)) {
057          permission = FsPermission.getDefault();
058        } else {
059          permission = new FsPermission(Short.parseShort(str, 8));
060        }
061        return permission;
062      }
063    
064      @SuppressWarnings({"unchecked", "deprecation"})
065      private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) {
066        Map json = new LinkedHashMap();
067        json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName());
068        json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString());
069        json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen());
070        json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner());
071        json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup());
072        json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission()));
073        json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime());
074        json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime());
075        json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize());
076        json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication());
077        return json;
078      }
079    
080      /**
081       * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON
082       * object.
083       *
084       * @param status FileSystemAccess file status.
085       *
086       * @return The JSON representation of the file status.
087       */
088      @SuppressWarnings({"unchecked", "deprecation"})
089      private static Map fileStatusToJSON(FileStatus status) {
090        Map json = new LinkedHashMap();
091        json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true));
092        return json;
093      }
094    
095      /**
096       * Converts a <code>FileChecksum</code> object into a JSON array
097       * object.
098       *
099       * @param checksum file checksum.
100       *
101       * @return The JSON representation of the file checksum.
102       */
103      @SuppressWarnings({"unchecked"})
104      private static Map fileChecksumToJSON(FileChecksum checksum) {
105        Map json = new LinkedHashMap();
106        json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName());
107        json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON,
108                 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes()));
109        json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength());
110        Map response = new LinkedHashMap();
111        response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json);
112        return response;
113      }
114    
115      /**
116       * Converts a <code>ContentSummary</code> object into a JSON array
117       * object.
118       *
119       * @param contentSummary the content summary
120       *
121       * @return The JSON representation of the content summary.
122       */
123      @SuppressWarnings({"unchecked"})
124      private static Map contentSummaryToJSON(ContentSummary contentSummary) {
125        Map json = new LinkedHashMap();
126        json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount());
127        json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount());
128        json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength());
129        json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota());
130        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed());
131        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota());
132        Map response = new LinkedHashMap();
133        response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json);
134        return response;
135      }
136    
137      /**
138       * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array
139       * object.
140       *
141       * @param status FileSystemAccess file status array.
142       * <code>SCHEME://HOST:PORT</code> in the file status.
143       *
144       * @return The JSON representation of the file status array.
145       */
146      @SuppressWarnings("unchecked")
147      private static Map fileStatusToJSON(FileStatus[] status) {
148        JSONArray json = new JSONArray();
149        if (status != null) {
150          for (FileStatus s : status) {
151            json.add(fileStatusToJSONRaw(s, false));
152          }
153        }
154        Map response = new LinkedHashMap();
155        Map temp = new LinkedHashMap();
156        temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json);
157        response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp);
158        return response;
159      }
160    
161      /**
162       * Converts an object into a Json Map with with one key-value entry.
163       * <p/>
164       * It assumes the given value is either a JSON primitive type or a
165       * <code>JsonAware</code> instance.
166       *
167       * @param name name for the key of the entry.
168       * @param value for the value of the entry.
169       *
170       * @return the JSON representation of the key-value pair.
171       */
172      @SuppressWarnings("unchecked")
173      private static JSONObject toJSON(String name, Object value) {
174        JSONObject json = new JSONObject();
175        json.put(name, value);
176        return json;
177      }
178    
179      /**
180       * Executor that performs an append FileSystemAccess files system operation.
181       */
182      public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> {
183        private InputStream is;
184        private Path path;
185    
186        /**
187         * Creates an Append executor.
188         *
189         * @param is input stream to append.
190         * @param path path of the file to append.
191         */
192        public FSAppend(InputStream is, String path) {
193          this.is = is;
194          this.path = new Path(path);
195        }
196    
197        /**
198         * Executes the filesystem operation.
199         *
200         * @param fs filesystem instance to use.
201         *
202         * @return void.
203         *
204         * @throws IOException thrown if an IO error occured.
205         */
206        @Override
207        public Void execute(FileSystem fs) throws IOException {
208          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
209          OutputStream os = fs.append(path, bufferSize);
210          IOUtils.copyBytes(is, os, bufferSize, true);
211          os.close();
212          return null;
213        }
214    
215      }
216    
217      /**
218       * Executor that performs a content-summary FileSystemAccess files system operation.
219       */
220      public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> {
221        private Path path;
222    
223        /**
224         * Creates a content-summary executor.
225         *
226         * @param path the path to retrieve the content-summary.
227         */
228        public FSContentSummary(String path) {
229          this.path = new Path(path);
230        }
231    
232        /**
233         * Executes the filesystem operation.
234         *
235         * @param fs filesystem instance to use.
236         *
237         * @return a Map object (JSON friendly) with the content-summary.
238         *
239         * @throws IOException thrown if an IO error occured.
240         */
241        @Override
242        public Map execute(FileSystem fs) throws IOException {
243          ContentSummary contentSummary = fs.getContentSummary(path);
244          return contentSummaryToJSON(contentSummary);
245        }
246    
247      }
248    
249      /**
250       * Executor that performs a create FileSystemAccess files system operation.
251       */
252      public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> {
253        private InputStream is;
254        private Path path;
255        private String permission;
256        private boolean override;
257        private short replication;
258        private long blockSize;
259    
260        /**
261         * Creates a Create executor.
262         *
263         * @param is input stream to for the file to create.
264         * @param path path of the file to create.
265         * @param perm permission for the file.
266         * @param override if the file should be overriden if it already exist.
267         * @param repl the replication factor for the file.
268         * @param blockSize the block size for the file.
269         */
270        public FSCreate(InputStream is, String path, String perm, boolean override, short repl, long blockSize) {
271          this.is = is;
272          this.path = new Path(path);
273          this.permission = perm;
274          this.override = override;
275          this.replication = repl;
276          this.blockSize = blockSize;
277        }
278    
279        /**
280         * Executes the filesystem operation.
281         *
282         * @param fs filesystem instance to use.
283         *
284         * @return The URI of the created file.
285         *
286         * @throws IOException thrown if an IO error occured.
287         */
288        @Override
289        public Void execute(FileSystem fs) throws IOException {
290          if (replication == -1) {
291            replication = fs.getDefaultReplication();
292          }
293          if (blockSize == -1) {
294            blockSize = fs.getDefaultBlockSize();
295          }
296          FsPermission fsPermission = getPermission(permission);
297          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
298          OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
299          IOUtils.copyBytes(is, os, bufferSize, true);
300          os.close();
301          return null;
302        }
303    
304      }
305    
306      /**
307       * Executor that performs a delete FileSystemAccess files system operation.
308       */
309      public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> {
310        private Path path;
311        private boolean recursive;
312    
313        /**
314         * Creates a Delete executor.
315         *
316         * @param path path to delete.
317         * @param recursive if the delete should be recursive or not.
318         */
319        public FSDelete(String path, boolean recursive) {
320          this.path = new Path(path);
321          this.recursive = recursive;
322        }
323    
324        /**
325         * Executes the filesystem operation.
326         *
327         * @param fs filesystem instance to use.
328         *
329         * @return <code>true</code> if the delete operation was successful,
330         *         <code>false</code> otherwise.
331         *
332         * @throws IOException thrown if an IO error occured.
333         */
334        @Override
335        public JSONObject execute(FileSystem fs) throws IOException {
336          boolean deleted = fs.delete(path, recursive);
337          return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted);
338        }
339    
340      }
341    
342      /**
343       * Executor that performs a file-checksum FileSystemAccess files system operation.
344       */
345      public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> {
346        private Path path;
347    
348        /**
349         * Creates a file-checksum executor.
350         *
351         * @param path the path to retrieve the checksum.
352         */
353        public FSFileChecksum(String path) {
354          this.path = new Path(path);
355        }
356    
357        /**
358         * Executes the filesystem operation.
359         *
360         * @param fs filesystem instance to use.
361         *
362         * @return a Map object (JSON friendly) with the file checksum.
363         *
364         * @throws IOException thrown if an IO error occured.
365         */
366        @Override
367        public Map execute(FileSystem fs) throws IOException {
368          FileChecksum checksum = fs.getFileChecksum(path);
369          return fileChecksumToJSON(checksum);
370        }
371    
372      }
373    
374      /**
375       * Executor that performs a file-status FileSystemAccess files system operation.
376       */
377      public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> {
378        private Path path;
379    
380        /**
381         * Creates a file-status executor.
382         *
383         * @param path the path to retrieve the status.
384         */
385        public FSFileStatus(String path) {
386          this.path = new Path(path);
387        }
388    
389        /**
390         * Executes the filesystem operation.
391         *
392         * @param fs filesystem instance to use.
393         *
394         * @return a Map object (JSON friendly) with the file status.
395         *
396         * @throws IOException thrown if an IO error occured.
397         */
398        @Override
399        public Map execute(FileSystem fs) throws IOException {
400          FileStatus status = fs.getFileStatus(path);
401          return fileStatusToJSON(status);
402        }
403    
404      }
405    
406      /**
407       * Executor that performs a home-dir FileSystemAccess files system operation.
408       */
409      public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> {
410    
411        /**
412         * Executes the filesystem operation.
413         *
414         * @param fs filesystem instance to use.
415         *
416         * @return a JSON object with the user home directory.
417         *
418         * @throws IOException thrown if an IO error occured.
419         */
420        @Override
421        @SuppressWarnings("unchecked")
422        public JSONObject execute(FileSystem fs) throws IOException {
423          Path homeDir = fs.getHomeDirectory();
424          JSONObject json = new JSONObject();
425          json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath());
426          return json;
427        }
428    
429      }
430    
431      /**
432       * Executor that performs a list-status FileSystemAccess files system operation.
433       */
434      public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter {
435        private Path path;
436        private PathFilter filter;
437    
438        /**
439         * Creates a list-status executor.
440         *
441         * @param path the directory to retrieve the status of its contents.
442         * @param filter glob filter to use.
443         *
444         * @throws IOException thrown if the filter expression is incorrect.
445         */
446        public FSListStatus(String path, String filter) throws IOException {
447          this.path = new Path(path);
448          this.filter = (filter == null) ? this : new GlobFilter(filter);
449        }
450    
451        /**
452         * Executes the filesystem operation.
453         *
454         * @param fs filesystem instance to use.
455         *
456         * @return a Map with the file status of the directory
457         *         contents.
458         *
459         * @throws IOException thrown if an IO error occured.
460         */
461        @Override
462        public Map execute(FileSystem fs) throws IOException {
463          FileStatus[] status = fs.listStatus(path, filter);
464          return fileStatusToJSON(status);
465        }
466    
467        @Override
468        public boolean accept(Path path) {
469          return true;
470        }
471    
472      }
473    
474      /**
475       * Executor that performs a mkdirs FileSystemAccess files system operation.
476       */
477      public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> {
478    
479        private Path path;
480        private String permission;
481    
482        /**
483         * Creates a mkdirs executor.
484         *
485         * @param path directory path to create.
486         * @param permission permission to use.
487         */
488        public FSMkdirs(String path, String permission) {
489          this.path = new Path(path);
490          this.permission = permission;
491        }
492    
493        /**
494         * Executes the filesystem operation.
495         *
496         * @param fs filesystem instance to use.
497         *
498         * @return <code>true</code> if the mkdirs operation was successful,
499         *         <code>false</code> otherwise.
500         *
501         * @throws IOException thrown if an IO error occured.
502         */
503        @Override
504        public JSONObject execute(FileSystem fs) throws IOException {
505          FsPermission fsPermission = getPermission(permission);
506          boolean mkdirs = fs.mkdirs(path, fsPermission);
507          return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs);
508        }
509    
510      }
511    
512      /**
513       * Executor that performs a open FileSystemAccess files system operation.
514       */
515      public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> {
516        private Path path;
517    
518        /**
519         * Creates a open executor.
520         *
521         * @param path file to open.
522         */
523        public FSOpen(String path) {
524          this.path = new Path(path);
525        }
526    
527        /**
528         * Executes the filesystem operation.
529         *
530         * @param fs filesystem instance to use.
531         *
532         * @return The inputstream of the file.
533         *
534         * @throws IOException thrown if an IO error occured.
535         */
536        @Override
537        public InputStream execute(FileSystem fs) throws IOException {
538          int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096);
539          return fs.open(path, bufferSize);
540        }
541    
542      }
543    
544      /**
545       * Executor that performs a rename FileSystemAccess files system operation.
546       */
547      public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> {
548        private Path path;
549        private Path toPath;
550    
551        /**
552         * Creates a rename executor.
553         *
554         * @param path path to rename.
555         * @param toPath new name.
556         */
557        public FSRename(String path, String toPath) {
558          this.path = new Path(path);
559          this.toPath = new Path(toPath);
560        }
561    
562        /**
563         * Executes the filesystem operation.
564         *
565         * @param fs filesystem instance to use.
566         *
567         * @return <code>true</code> if the rename operation was successful,
568         *         <code>false</code> otherwise.
569         *
570         * @throws IOException thrown if an IO error occured.
571         */
572        @Override
573        public JSONObject execute(FileSystem fs) throws IOException {
574          boolean renamed = fs.rename(path, toPath);
575          return toJSON(HttpFSFileSystem.RENAME_JSON, renamed);
576        }
577    
578      }
579    
580      /**
581       * Executor that performs a set-owner FileSystemAccess files system operation.
582       */
583      public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> {
584        private Path path;
585        private String owner;
586        private String group;
587    
588        /**
589         * Creates a set-owner executor.
590         *
591         * @param path the path to set the owner.
592         * @param owner owner to set.
593         * @param group group to set.
594         */
595        public FSSetOwner(String path, String owner, String group) {
596          this.path = new Path(path);
597          this.owner = owner;
598          this.group = group;
599        }
600    
601        /**
602         * Executes the filesystem operation.
603         *
604         * @param fs filesystem instance to use.
605         *
606         * @return void.
607         *
608         * @throws IOException thrown if an IO error occured.
609         */
610        @Override
611        public Void execute(FileSystem fs) throws IOException {
612          fs.setOwner(path, owner, group);
613          return null;
614        }
615    
616      }
617    
618      /**
619       * Executor that performs a set-permission FileSystemAccess files system operation.
620       */
621      public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> {
622    
623        private Path path;
624        private String permission;
625    
626        /**
627         * Creates a set-permission executor.
628         *
629         * @param path path to set the permission.
630         * @param permission permission to set.
631         */
632        public FSSetPermission(String path, String permission) {
633          this.path = new Path(path);
634          this.permission = permission;
635        }
636    
637        /**
638         * Executes the filesystem operation.
639         *
640         * @param fs filesystem instance to use.
641         *
642         * @return void.
643         *
644         * @throws IOException thrown if an IO error occured.
645         */
646        @Override
647        public Void execute(FileSystem fs) throws IOException {
648          FsPermission fsPermission = getPermission(permission);
649          fs.setPermission(path, fsPermission);
650          return null;
651        }
652    
653      }
654    
655      /**
656       * Executor that performs a set-replication FileSystemAccess files system operation.
657       */
658      public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> {
659        private Path path;
660        private short replication;
661    
662        /**
663         * Creates a set-replication executor.
664         *
665         * @param path path to set the replication factor.
666         * @param replication replication factor to set.
667         */
668        public FSSetReplication(String path, short replication) {
669          this.path = new Path(path);
670          this.replication = replication;
671        }
672    
673        /**
674         * Executes the filesystem operation.
675         *
676         * @param fs filesystem instance to use.
677         *
678         * @return <code>true</code> if the replication value was set,
679         *         <code>false</code> otherwise.
680         *
681         * @throws IOException thrown if an IO error occured.
682         */
683        @Override
684        @SuppressWarnings("unchecked")
685        public JSONObject execute(FileSystem fs) throws IOException {
686          boolean ret = fs.setReplication(path, replication);
687          JSONObject json = new JSONObject();
688          json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret);
689          return json;
690        }
691    
692      }
693    
694      /**
695       * Executor that performs a set-times FileSystemAccess files system operation.
696       */
697      public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> {
698        private Path path;
699        private long mTime;
700        private long aTime;
701    
702        /**
703         * Creates a set-times executor.
704         *
705         * @param path path to set the times.
706         * @param mTime modified time to set.
707         * @param aTime access time to set.
708         */
709        public FSSetTimes(String path, long mTime, long aTime) {
710          this.path = new Path(path);
711          this.mTime = mTime;
712          this.aTime = aTime;
713        }
714    
715        /**
716         * Executes the filesystem operation.
717         *
718         * @param fs filesystem instance to use.
719         *
720         * @return void.
721         *
722         * @throws IOException thrown if an IO error occured.
723         */
724        @Override
725        public Void execute(FileSystem fs) throws IOException {
726          fs.setTimes(path, mTime, aTime);
727          return null;
728        }
729    
730      }
731    
732    }