001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs.http.server;
019    
020    import org.apache.hadoop.classification.InterfaceAudience;
021    import org.apache.hadoop.fs.ContentSummary;
022    import org.apache.hadoop.fs.FileChecksum;
023    import org.apache.hadoop.fs.FileStatus;
024    import org.apache.hadoop.fs.FileSystem;
025    import org.apache.hadoop.fs.GlobFilter;
026    import org.apache.hadoop.fs.Path;
027    import org.apache.hadoop.fs.PathFilter;
028    import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
029    import org.apache.hadoop.fs.permission.FsPermission;
030    import org.apache.hadoop.io.IOUtils;
031    import org.apache.hadoop.lib.service.FileSystemAccess;
032    import org.json.simple.JSONArray;
033    import org.json.simple.JSONObject;
034    
035    import java.io.IOException;
036    import java.io.InputStream;
037    import java.io.OutputStream;
038    import java.util.LinkedHashMap;
039    import java.util.Map;
040    
041    /**
042     * FileSystem operation executors used by {@link HttpFSServer}.
043     */
044    @InterfaceAudience.Private
045    public class FSOperations {
046    
047      @SuppressWarnings({"unchecked", "deprecation"})
048      private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) {
049        Map json = new LinkedHashMap();
050        json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName());
051        json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString());
052        json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen());
053        json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner());
054        json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup());
055        json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission()));
056        json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime());
057        json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime());
058        json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize());
059        json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication());
060        return json;
061      }
062    
063      /**
064       * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON
065       * object.
066       *
067       * @param status FileSystemAccess file status.
068       *
069       * @return The JSON representation of the file status.
070       */
071      @SuppressWarnings({"unchecked", "deprecation"})
072      private static Map fileStatusToJSON(FileStatus status) {
073        Map json = new LinkedHashMap();
074        json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true));
075        return json;
076      }
077    
078      /**
079       * Converts a <code>FileChecksum</code> object into a JSON array
080       * object.
081       *
082       * @param checksum file checksum.
083       *
084       * @return The JSON representation of the file checksum.
085       */
086      @SuppressWarnings({"unchecked"})
087      private static Map fileChecksumToJSON(FileChecksum checksum) {
088        Map json = new LinkedHashMap();
089        json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName());
090        json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON,
091                 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes()));
092        json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength());
093        Map response = new LinkedHashMap();
094        response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json);
095        return response;
096      }
097    
098      /**
099       * Converts a <code>ContentSummary</code> object into a JSON array
100       * object.
101       *
102       * @param contentSummary the content summary
103       *
104       * @return The JSON representation of the content summary.
105       */
106      @SuppressWarnings({"unchecked"})
107      private static Map contentSummaryToJSON(ContentSummary contentSummary) {
108        Map json = new LinkedHashMap();
109        json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount());
110        json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount());
111        json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength());
112        json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota());
113        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed());
114        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota());
115        Map response = new LinkedHashMap();
116        response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json);
117        return response;
118      }
119    
120      /**
121       * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array
122       * object.
123       *
124       * @param status FileSystemAccess file status array.
125       * <code>SCHEME://HOST:PORT</code> in the file status.
126       *
127       * @return The JSON representation of the file status array.
128       */
129      @SuppressWarnings("unchecked")
130      private static Map fileStatusToJSON(FileStatus[] status) {
131        JSONArray json = new JSONArray();
132        if (status != null) {
133          for (FileStatus s : status) {
134            json.add(fileStatusToJSONRaw(s, false));
135          }
136        }
137        Map response = new LinkedHashMap();
138        Map temp = new LinkedHashMap();
139        temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json);
140        response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp);
141        return response;
142      }
143    
144      /**
145       * Converts an object into a Json Map with with one key-value entry.
146       * <p/>
147       * It assumes the given value is either a JSON primitive type or a
148       * <code>JsonAware</code> instance.
149       *
150       * @param name name for the key of the entry.
151       * @param value for the value of the entry.
152       *
153       * @return the JSON representation of the key-value pair.
154       */
155      @SuppressWarnings("unchecked")
156      private static JSONObject toJSON(String name, Object value) {
157        JSONObject json = new JSONObject();
158        json.put(name, value);
159        return json;
160      }
161    
162      /**
163       * Executor that performs an append FileSystemAccess files system operation.
164       */
165      @InterfaceAudience.Private
166      public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> {
167        private InputStream is;
168        private Path path;
169    
170        /**
171         * Creates an Append executor.
172         *
173         * @param is input stream to append.
174         * @param path path of the file to append.
175         */
176        public FSAppend(InputStream is, String path) {
177          this.is = is;
178          this.path = new Path(path);
179        }
180    
181        /**
182         * Executes the filesystem operation.
183         *
184         * @param fs filesystem instance to use.
185         *
186         * @return void.
187         *
188         * @throws IOException thrown if an IO error occured.
189         */
190        @Override
191        public Void execute(FileSystem fs) throws IOException {
192          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
193          OutputStream os = fs.append(path, bufferSize);
194          IOUtils.copyBytes(is, os, bufferSize, true);
195          os.close();
196          return null;
197        }
198    
199      }
200    
201      /**
202       * Executor that performs an append FileSystemAccess files system operation.
203       */
204      @InterfaceAudience.Private
205      public static class FSConcat implements FileSystemAccess.FileSystemExecutor<Void> {
206        private Path path;
207        private Path[] sources;
208    
209        /**
210         * Creates a Concat executor.
211         *
212         * @param path target path to concat to.
213         * @param sources comma seperated absolute paths to use as sources.
214         */
215        public FSConcat(String path, String[] sources) {
216          this.sources = new Path[sources.length];
217    
218          for(int i = 0; i < sources.length; i++) {
219            this.sources[i] = new Path(sources[i]);
220          }
221    
222          this.path = new Path(path);
223        }
224    
225        /**
226         * Executes the filesystem operation.
227         *
228         * @param fs filesystem instance to use.
229         *
230         * @return void.
231         *
232         * @throws IOException thrown if an IO error occured.
233         */
234        @Override
235        public Void execute(FileSystem fs) throws IOException {
236          fs.concat(path, sources);
237          return null;
238        }
239    
240      }
241    
242      /**
243       * Executor that performs a content-summary FileSystemAccess files system operation.
244       */
245      @InterfaceAudience.Private
246      public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> {
247        private Path path;
248    
249        /**
250         * Creates a content-summary executor.
251         *
252         * @param path the path to retrieve the content-summary.
253         */
254        public FSContentSummary(String path) {
255          this.path = new Path(path);
256        }
257    
258        /**
259         * Executes the filesystem operation.
260         *
261         * @param fs filesystem instance to use.
262         *
263         * @return a Map object (JSON friendly) with the content-summary.
264         *
265         * @throws IOException thrown if an IO error occured.
266         */
267        @Override
268        public Map execute(FileSystem fs) throws IOException {
269          ContentSummary contentSummary = fs.getContentSummary(path);
270          return contentSummaryToJSON(contentSummary);
271        }
272    
273      }
274    
275      /**
276       * Executor that performs a create FileSystemAccess files system operation.
277       */
278      @InterfaceAudience.Private
279      public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> {
280        private InputStream is;
281        private Path path;
282        private short permission;
283        private boolean override;
284        private short replication;
285        private long blockSize;
286    
287        /**
288         * Creates a Create executor.
289         *
290         * @param is input stream to for the file to create.
291         * @param path path of the file to create.
292         * @param perm permission for the file.
293         * @param override if the file should be overriden if it already exist.
294         * @param repl the replication factor for the file.
295         * @param blockSize the block size for the file.
296         */
297        public FSCreate(InputStream is, String path, short perm, boolean override,
298                        short repl, long blockSize) {
299          this.is = is;
300          this.path = new Path(path);
301          this.permission = perm;
302          this.override = override;
303          this.replication = repl;
304          this.blockSize = blockSize;
305        }
306    
307        /**
308         * Executes the filesystem operation.
309         *
310         * @param fs filesystem instance to use.
311         *
312         * @return The URI of the created file.
313         *
314         * @throws IOException thrown if an IO error occured.
315         */
316        @Override
317        public Void execute(FileSystem fs) throws IOException {
318          if (replication == -1) {
319            replication = fs.getDefaultReplication(path);
320          }
321          if (blockSize == -1) {
322            blockSize = fs.getDefaultBlockSize(path);
323          }
324          FsPermission fsPermission = new FsPermission(permission);
325          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
326          OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
327          IOUtils.copyBytes(is, os, bufferSize, true);
328          os.close();
329          return null;
330        }
331    
332      }
333    
334      /**
335       * Executor that performs a delete FileSystemAccess files system operation.
336       */
337      @InterfaceAudience.Private
338      public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> {
339        private Path path;
340        private boolean recursive;
341    
342        /**
343         * Creates a Delete executor.
344         *
345         * @param path path to delete.
346         * @param recursive if the delete should be recursive or not.
347         */
348        public FSDelete(String path, boolean recursive) {
349          this.path = new Path(path);
350          this.recursive = recursive;
351        }
352    
353        /**
354         * Executes the filesystem operation.
355         *
356         * @param fs filesystem instance to use.
357         *
358         * @return <code>true</code> if the delete operation was successful,
359         *         <code>false</code> otherwise.
360         *
361         * @throws IOException thrown if an IO error occured.
362         */
363        @Override
364        public JSONObject execute(FileSystem fs) throws IOException {
365          boolean deleted = fs.delete(path, recursive);
366          return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted);
367        }
368    
369      }
370    
371      /**
372       * Executor that performs a file-checksum FileSystemAccess files system operation.
373       */
374      @InterfaceAudience.Private
375      public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> {
376        private Path path;
377    
378        /**
379         * Creates a file-checksum executor.
380         *
381         * @param path the path to retrieve the checksum.
382         */
383        public FSFileChecksum(String path) {
384          this.path = new Path(path);
385        }
386    
387        /**
388         * Executes the filesystem operation.
389         *
390         * @param fs filesystem instance to use.
391         *
392         * @return a Map object (JSON friendly) with the file checksum.
393         *
394         * @throws IOException thrown if an IO error occured.
395         */
396        @Override
397        public Map execute(FileSystem fs) throws IOException {
398          FileChecksum checksum = fs.getFileChecksum(path);
399          return fileChecksumToJSON(checksum);
400        }
401    
402      }
403    
404      /**
405       * Executor that performs a file-status FileSystemAccess files system operation.
406       */
407      @InterfaceAudience.Private
408      public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> {
409        private Path path;
410    
411        /**
412         * Creates a file-status executor.
413         *
414         * @param path the path to retrieve the status.
415         */
416        public FSFileStatus(String path) {
417          this.path = new Path(path);
418        }
419    
420        /**
421         * Executes the filesystem operation.
422         *
423         * @param fs filesystem instance to use.
424         *
425         * @return a Map object (JSON friendly) with the file status.
426         *
427         * @throws IOException thrown if an IO error occured.
428         */
429        @Override
430        public Map execute(FileSystem fs) throws IOException {
431          FileStatus status = fs.getFileStatus(path);
432          return fileStatusToJSON(status);
433        }
434    
435      }
436    
437      /**
438       * Executor that performs a home-dir FileSystemAccess files system operation.
439       */
440      @InterfaceAudience.Private
441      public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> {
442    
443        /**
444         * Executes the filesystem operation.
445         *
446         * @param fs filesystem instance to use.
447         *
448         * @return a JSON object with the user home directory.
449         *
450         * @throws IOException thrown if an IO error occured.
451         */
452        @Override
453        @SuppressWarnings("unchecked")
454        public JSONObject execute(FileSystem fs) throws IOException {
455          Path homeDir = fs.getHomeDirectory();
456          JSONObject json = new JSONObject();
457          json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath());
458          return json;
459        }
460    
461      }
462    
463      /**
464       * Executor that performs a list-status FileSystemAccess files system operation.
465       */
466      @InterfaceAudience.Private
467      public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter {
468        private Path path;
469        private PathFilter filter;
470    
471        /**
472         * Creates a list-status executor.
473         *
474         * @param path the directory to retrieve the status of its contents.
475         * @param filter glob filter to use.
476         *
477         * @throws IOException thrown if the filter expression is incorrect.
478         */
479        public FSListStatus(String path, String filter) throws IOException {
480          this.path = new Path(path);
481          this.filter = (filter == null) ? this : new GlobFilter(filter);
482        }
483    
484        /**
485         * Executes the filesystem operation.
486         *
487         * @param fs filesystem instance to use.
488         *
489         * @return a Map with the file status of the directory
490         *         contents.
491         *
492         * @throws IOException thrown if an IO error occured.
493         */
494        @Override
495        public Map execute(FileSystem fs) throws IOException {
496          FileStatus[] status = fs.listStatus(path, filter);
497          return fileStatusToJSON(status);
498        }
499    
500        @Override
501        public boolean accept(Path path) {
502          return true;
503        }
504    
505      }
506    
507      /**
508       * Executor that performs a mkdirs FileSystemAccess files system operation.
509       */
510      @InterfaceAudience.Private
511      public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> {
512    
513        private Path path;
514        private short permission;
515    
516        /**
517         * Creates a mkdirs executor.
518         *
519         * @param path directory path to create.
520         * @param permission permission to use.
521         */
522        public FSMkdirs(String path, short permission) {
523          this.path = new Path(path);
524          this.permission = permission;
525        }
526    
527        /**
528         * Executes the filesystem operation.
529         *
530         * @param fs filesystem instance to use.
531         *
532         * @return <code>true</code> if the mkdirs operation was successful,
533         *         <code>false</code> otherwise.
534         *
535         * @throws IOException thrown if an IO error occured.
536         */
537        @Override
538        public JSONObject execute(FileSystem fs) throws IOException {
539          FsPermission fsPermission = new FsPermission(permission);
540          boolean mkdirs = fs.mkdirs(path, fsPermission);
541          return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs);
542        }
543    
544      }
545    
546      /**
547       * Executor that performs a open FileSystemAccess files system operation.
548       */
549      @InterfaceAudience.Private
550      public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> {
551        private Path path;
552    
553        /**
554         * Creates a open executor.
555         *
556         * @param path file to open.
557         */
558        public FSOpen(String path) {
559          this.path = new Path(path);
560        }
561    
562        /**
563         * Executes the filesystem operation.
564         *
565         * @param fs filesystem instance to use.
566         *
567         * @return The inputstream of the file.
568         *
569         * @throws IOException thrown if an IO error occured.
570         */
571        @Override
572        public InputStream execute(FileSystem fs) throws IOException {
573          int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096);
574          return fs.open(path, bufferSize);
575        }
576    
577      }
578    
579      /**
580       * Executor that performs a rename FileSystemAccess files system operation.
581       */
582      @InterfaceAudience.Private
583      public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> {
584        private Path path;
585        private Path toPath;
586    
587        /**
588         * Creates a rename executor.
589         *
590         * @param path path to rename.
591         * @param toPath new name.
592         */
593        public FSRename(String path, String toPath) {
594          this.path = new Path(path);
595          this.toPath = new Path(toPath);
596        }
597    
598        /**
599         * Executes the filesystem operation.
600         *
601         * @param fs filesystem instance to use.
602         *
603         * @return <code>true</code> if the rename operation was successful,
604         *         <code>false</code> otherwise.
605         *
606         * @throws IOException thrown if an IO error occured.
607         */
608        @Override
609        public JSONObject execute(FileSystem fs) throws IOException {
610          boolean renamed = fs.rename(path, toPath);
611          return toJSON(HttpFSFileSystem.RENAME_JSON, renamed);
612        }
613    
614      }
615    
616      /**
617       * Executor that performs a set-owner FileSystemAccess files system operation.
618       */
619      @InterfaceAudience.Private
620      public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> {
621        private Path path;
622        private String owner;
623        private String group;
624    
625        /**
626         * Creates a set-owner executor.
627         *
628         * @param path the path to set the owner.
629         * @param owner owner to set.
630         * @param group group to set.
631         */
632        public FSSetOwner(String path, String owner, String group) {
633          this.path = new Path(path);
634          this.owner = owner;
635          this.group = group;
636        }
637    
638        /**
639         * Executes the filesystem operation.
640         *
641         * @param fs filesystem instance to use.
642         *
643         * @return void.
644         *
645         * @throws IOException thrown if an IO error occured.
646         */
647        @Override
648        public Void execute(FileSystem fs) throws IOException {
649          fs.setOwner(path, owner, group);
650          return null;
651        }
652    
653      }
654    
655      /**
656       * Executor that performs a set-permission FileSystemAccess files system operation.
657       */
658      @InterfaceAudience.Private
659      public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> {
660    
661        private Path path;
662        private short permission;
663    
664        /**
665         * Creates a set-permission executor.
666         *
667         * @param path path to set the permission.
668         * @param permission permission to set.
669         */
670        public FSSetPermission(String path, short permission) {
671          this.path = new Path(path);
672          this.permission = permission;
673        }
674    
675        /**
676         * Executes the filesystem operation.
677         *
678         * @param fs filesystem instance to use.
679         *
680         * @return void.
681         *
682         * @throws IOException thrown if an IO error occured.
683         */
684        @Override
685        public Void execute(FileSystem fs) throws IOException {
686          FsPermission fsPermission = new FsPermission(permission);
687          fs.setPermission(path, fsPermission);
688          return null;
689        }
690    
691      }
692    
693      /**
694       * Executor that performs a set-replication FileSystemAccess files system operation.
695       */
696      @InterfaceAudience.Private
697      public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> {
698        private Path path;
699        private short replication;
700    
701        /**
702         * Creates a set-replication executor.
703         *
704         * @param path path to set the replication factor.
705         * @param replication replication factor to set.
706         */
707        public FSSetReplication(String path, short replication) {
708          this.path = new Path(path);
709          this.replication = replication;
710        }
711    
712        /**
713         * Executes the filesystem operation.
714         *
715         * @param fs filesystem instance to use.
716         *
717         * @return <code>true</code> if the replication value was set,
718         *         <code>false</code> otherwise.
719         *
720         * @throws IOException thrown if an IO error occured.
721         */
722        @Override
723        @SuppressWarnings("unchecked")
724        public JSONObject execute(FileSystem fs) throws IOException {
725          boolean ret = fs.setReplication(path, replication);
726          JSONObject json = new JSONObject();
727          json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret);
728          return json;
729        }
730    
731      }
732    
733      /**
734       * Executor that performs a set-times FileSystemAccess files system operation.
735       */
736      @InterfaceAudience.Private
737      public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> {
738        private Path path;
739        private long mTime;
740        private long aTime;
741    
742        /**
743         * Creates a set-times executor.
744         *
745         * @param path path to set the times.
746         * @param mTime modified time to set.
747         * @param aTime access time to set.
748         */
749        public FSSetTimes(String path, long mTime, long aTime) {
750          this.path = new Path(path);
751          this.mTime = mTime;
752          this.aTime = aTime;
753        }
754    
755        /**
756         * Executes the filesystem operation.
757         *
758         * @param fs filesystem instance to use.
759         *
760         * @return void.
761         *
762         * @throws IOException thrown if an IO error occured.
763         */
764        @Override
765        public Void execute(FileSystem fs) throws IOException {
766          fs.setTimes(path, mTime, aTime);
767          return null;
768        }
769    
770      }
771    
772    }