001    package org.apache.hadoop.fs.http.server;
002    
003    import org.apache.hadoop.fs.ContentSummary;
004    import org.apache.hadoop.fs.FileChecksum;
005    import org.apache.hadoop.fs.FileStatus;
006    import org.apache.hadoop.fs.FileSystem;
007    import org.apache.hadoop.fs.GlobFilter;
008    import org.apache.hadoop.fs.Path;
009    import org.apache.hadoop.fs.PathFilter;
010    import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
011    import org.apache.hadoop.fs.permission.FsPermission;
012    import org.apache.hadoop.io.IOUtils;
013    import org.apache.hadoop.lib.service.FileSystemAccess;
014    import org.json.simple.JSONArray;
015    import org.json.simple.JSONObject;
016    
017    import java.io.IOException;
018    import java.io.InputStream;
019    import java.io.OutputStream;
020    import java.util.LinkedHashMap;
021    import java.util.Map;
022    
023    /**
024     * FileSystem operation executors used by {@link HttpFSServer}.
025     */
026    public class FSOperations {
027    
028      /**
029       * Converts a Unix permission octal & symbolic representation
030       * (i.e. 655 or -rwxr--r--) into a FileSystemAccess permission.
031       *
032       * @param str Unix permission symbolic representation.
033       *
034       * @return the FileSystemAccess permission. If the given string was
035       *         'default', it returns <code>FsPermission.getDefault()</code>.
036       */
037      private static FsPermission getPermission(String str) {
038        FsPermission permission;
039        if (str.equals(HttpFSFileSystem.DEFAULT_PERMISSION)) {
040          permission = FsPermission.getDefault();
041        } else if (str.length() == 3) {
042          permission = new FsPermission(Short.parseShort(str, 8));
043        } else {
044          permission = FsPermission.valueOf(str);
045        }
046        return permission;
047      }
048    
049      @SuppressWarnings({"unchecked", "deprecation"})
050      private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) {
051        Map json = new LinkedHashMap();
052        json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName());
053        json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString());
054        json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen());
055        json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner());
056        json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup());
057        json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission()));
058        json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime());
059        json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime());
060        json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize());
061        json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication());
062        return json;
063      }
064    
065      /**
066       * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON
067       * object.
068       *
069       * @param status FileSystemAccess file status.
070       *
071       * @return The JSON representation of the file status.
072       */
073      @SuppressWarnings({"unchecked", "deprecation"})
074      private static Map fileStatusToJSON(FileStatus status) {
075        Map json = new LinkedHashMap();
076        json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true));
077        return json;
078      }
079    
080      /**
081       * Converts a <code>FileChecksum</code> object into a JSON array
082       * object.
083       *
084       * @param checksum file checksum.
085       *
086       * @return The JSON representation of the file checksum.
087       */
088      @SuppressWarnings({"unchecked"})
089      private static Map fileChecksumToJSON(FileChecksum checksum) {
090        Map json = new LinkedHashMap();
091        json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName());
092        json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON,
093                 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes()));
094        json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength());
095        Map response = new LinkedHashMap();
096        response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json);
097        return response;
098      }
099    
100      /**
101       * Converts a <code>ContentSummary</code> object into a JSON array
102       * object.
103       *
104       * @param contentSummary the content summary
105       *
106       * @return The JSON representation of the content summary.
107       */
108      @SuppressWarnings({"unchecked"})
109      private static Map contentSummaryToJSON(ContentSummary contentSummary) {
110        Map json = new LinkedHashMap();
111        json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount());
112        json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount());
113        json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength());
114        json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota());
115        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed());
116        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota());
117        Map response = new LinkedHashMap();
118        response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json);
119        return response;
120      }
121    
122      /**
123       * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array
124       * object.
125       *
126       * @param status FileSystemAccess file status array.
127       * <code>SCHEME://HOST:PORT</code> in the file status.
128       *
129       * @return The JSON representation of the file status array.
130       */
131      @SuppressWarnings("unchecked")
132      private static Map fileStatusToJSON(FileStatus[] status) {
133        JSONArray json = new JSONArray();
134        if (status != null) {
135          for (FileStatus s : status) {
136            json.add(fileStatusToJSONRaw(s, false));
137          }
138        }
139        Map response = new LinkedHashMap();
140        Map temp = new LinkedHashMap();
141        temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json);
142        response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp);
143        return response;
144      }
145    
146      /**
147       * Converts an object into a Json Map with with one key-value entry.
148       * <p/>
149       * It assumes the given value is either a JSON primitive type or a
150       * <code>JsonAware</code> instance.
151       *
152       * @param name name for the key of the entry.
153       * @param value for the value of the entry.
154       *
155       * @return the JSON representation of the key-value pair.
156       */
157      @SuppressWarnings("unchecked")
158      private static JSONObject toJSON(String name, Object value) {
159        JSONObject json = new JSONObject();
160        json.put(name, value);
161        return json;
162      }
163    
164      /**
165       * Executor that performs an append FileSystemAccess files system operation.
166       */
167      public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> {
168        private InputStream is;
169        private Path path;
170    
171        /**
172         * Creates an Append executor.
173         *
174         * @param is input stream to append.
175         * @param path path of the file to append.
176         */
177        public FSAppend(InputStream is, String path) {
178          this.is = is;
179          this.path = new Path(path);
180        }
181    
182        /**
183         * Executes the filesystem operation.
184         *
185         * @param fs filesystem instance to use.
186         *
187         * @return void.
188         *
189         * @throws IOException thrown if an IO error occured.
190         */
191        @Override
192        public Void execute(FileSystem fs) throws IOException {
193          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
194          OutputStream os = fs.append(path, bufferSize);
195          IOUtils.copyBytes(is, os, bufferSize, true);
196          os.close();
197          return null;
198        }
199    
200      }
201    
202      /**
203       * Executor that performs a content-summary FileSystemAccess files system operation.
204       */
205      public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> {
206        private Path path;
207    
208        /**
209         * Creates a content-summary executor.
210         *
211         * @param path the path to retrieve the content-summary.
212         */
213        public FSContentSummary(String path) {
214          this.path = new Path(path);
215        }
216    
217        /**
218         * Executes the filesystem operation.
219         *
220         * @param fs filesystem instance to use.
221         *
222         * @return a Map object (JSON friendly) with the content-summary.
223         *
224         * @throws IOException thrown if an IO error occured.
225         */
226        @Override
227        public Map execute(FileSystem fs) throws IOException {
228          ContentSummary contentSummary = fs.getContentSummary(path);
229          return contentSummaryToJSON(contentSummary);
230        }
231    
232      }
233    
234      /**
235       * Executor that performs a create FileSystemAccess files system operation.
236       */
237      public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> {
238        private InputStream is;
239        private Path path;
240        private String permission;
241        private boolean override;
242        private short replication;
243        private long blockSize;
244    
245        /**
246         * Creates a Create executor.
247         *
248         * @param is input stream to for the file to create.
249         * @param path path of the file to create.
250         * @param perm permission for the file.
251         * @param override if the file should be overriden if it already exist.
252         * @param repl the replication factor for the file.
253         * @param blockSize the block size for the file.
254         */
255        public FSCreate(InputStream is, String path, String perm, boolean override, short repl, long blockSize) {
256          this.is = is;
257          this.path = new Path(path);
258          this.permission = perm;
259          this.override = override;
260          this.replication = repl;
261          this.blockSize = blockSize;
262        }
263    
264        /**
265         * Executes the filesystem operation.
266         *
267         * @param fs filesystem instance to use.
268         *
269         * @return The URI of the created file.
270         *
271         * @throws IOException thrown if an IO error occured.
272         */
273        @Override
274        public Void execute(FileSystem fs) throws IOException {
275          if (replication == -1) {
276            replication = fs.getDefaultReplication();
277          }
278          if (blockSize == -1) {
279            blockSize = fs.getDefaultBlockSize();
280          }
281          FsPermission fsPermission = getPermission(permission);
282          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
283          OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
284          IOUtils.copyBytes(is, os, bufferSize, true);
285          os.close();
286          return null;
287        }
288    
289      }
290    
291      /**
292       * Executor that performs a delete FileSystemAccess files system operation.
293       */
294      public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> {
295        private Path path;
296        private boolean recursive;
297    
298        /**
299         * Creates a Delete executor.
300         *
301         * @param path path to delete.
302         * @param recursive if the delete should be recursive or not.
303         */
304        public FSDelete(String path, boolean recursive) {
305          this.path = new Path(path);
306          this.recursive = recursive;
307        }
308    
309        /**
310         * Executes the filesystem operation.
311         *
312         * @param fs filesystem instance to use.
313         *
314         * @return <code>true</code> if the delete operation was successful,
315         *         <code>false</code> otherwise.
316         *
317         * @throws IOException thrown if an IO error occured.
318         */
319        @Override
320        public JSONObject execute(FileSystem fs) throws IOException {
321          boolean deleted = fs.delete(path, recursive);
322          return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted);
323        }
324    
325      }
326    
327      /**
328       * Executor that performs a file-checksum FileSystemAccess files system operation.
329       */
330      public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> {
331        private Path path;
332    
333        /**
334         * Creates a file-checksum executor.
335         *
336         * @param path the path to retrieve the checksum.
337         */
338        public FSFileChecksum(String path) {
339          this.path = new Path(path);
340        }
341    
342        /**
343         * Executes the filesystem operation.
344         *
345         * @param fs filesystem instance to use.
346         *
347         * @return a Map object (JSON friendly) with the file checksum.
348         *
349         * @throws IOException thrown if an IO error occured.
350         */
351        @Override
352        public Map execute(FileSystem fs) throws IOException {
353          FileChecksum checksum = fs.getFileChecksum(path);
354          return fileChecksumToJSON(checksum);
355        }
356    
357      }
358    
359      /**
360       * Executor that performs a file-status FileSystemAccess files system operation.
361       */
362      public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> {
363        private Path path;
364    
365        /**
366         * Creates a file-status executor.
367         *
368         * @param path the path to retrieve the status.
369         */
370        public FSFileStatus(String path) {
371          this.path = new Path(path);
372        }
373    
374        /**
375         * Executes the filesystem operation.
376         *
377         * @param fs filesystem instance to use.
378         *
379         * @return a Map object (JSON friendly) with the file status.
380         *
381         * @throws IOException thrown if an IO error occured.
382         */
383        @Override
384        public Map execute(FileSystem fs) throws IOException {
385          FileStatus status = fs.getFileStatus(path);
386          return fileStatusToJSON(status);
387        }
388    
389      }
390    
391      /**
392       * Executor that performs a home-dir FileSystemAccess files system operation.
393       */
394      public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> {
395    
396        /**
397         * Executes the filesystem operation.
398         *
399         * @param fs filesystem instance to use.
400         *
401         * @return a JSON object with the user home directory.
402         *
403         * @throws IOException thrown if an IO error occured.
404         */
405        @Override
406        @SuppressWarnings("unchecked")
407        public JSONObject execute(FileSystem fs) throws IOException {
408          Path homeDir = fs.getHomeDirectory();
409          JSONObject json = new JSONObject();
410          json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath());
411          return json;
412        }
413    
414      }
415    
416      /**
417       * Executor that performs a list-status FileSystemAccess files system operation.
418       */
419      public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter {
420        private Path path;
421        private PathFilter filter;
422    
423        /**
424         * Creates a list-status executor.
425         *
426         * @param path the directory to retrieve the status of its contents.
427         * @param filter glob filter to use.
428         *
429         * @throws IOException thrown if the filter expression is incorrect.
430         */
431        public FSListStatus(String path, String filter) throws IOException {
432          this.path = new Path(path);
433          this.filter = (filter == null) ? this : new GlobFilter(filter);
434        }
435    
436        /**
437         * Executes the filesystem operation.
438         *
439         * @param fs filesystem instance to use.
440         *
441         * @return a Map with the file status of the directory
442         *         contents.
443         *
444         * @throws IOException thrown if an IO error occured.
445         */
446        @Override
447        public Map execute(FileSystem fs) throws IOException {
448          FileStatus[] status = fs.listStatus(path, filter);
449          return fileStatusToJSON(status);
450        }
451    
452        @Override
453        public boolean accept(Path path) {
454          return true;
455        }
456    
457      }
458    
459      /**
460       * Executor that performs a mkdirs FileSystemAccess files system operation.
461       */
462      public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> {
463    
464        private Path path;
465        private String permission;
466    
467        /**
468         * Creates a mkdirs executor.
469         *
470         * @param path directory path to create.
471         * @param permission permission to use.
472         */
473        public FSMkdirs(String path, String permission) {
474          this.path = new Path(path);
475          this.permission = permission;
476        }
477    
478        /**
479         * Executes the filesystem operation.
480         *
481         * @param fs filesystem instance to use.
482         *
483         * @return <code>true</code> if the mkdirs operation was successful,
484         *         <code>false</code> otherwise.
485         *
486         * @throws IOException thrown if an IO error occured.
487         */
488        @Override
489        public JSONObject execute(FileSystem fs) throws IOException {
490          FsPermission fsPermission = getPermission(permission);
491          boolean mkdirs = fs.mkdirs(path, fsPermission);
492          return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs);
493        }
494    
495      }
496    
497      /**
498       * Executor that performs a open FileSystemAccess files system operation.
499       */
500      public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> {
501        private Path path;
502    
503        /**
504         * Creates a open executor.
505         *
506         * @param path file to open.
507         */
508        public FSOpen(String path) {
509          this.path = new Path(path);
510        }
511    
512        /**
513         * Executes the filesystem operation.
514         *
515         * @param fs filesystem instance to use.
516         *
517         * @return The inputstream of the file.
518         *
519         * @throws IOException thrown if an IO error occured.
520         */
521        @Override
522        public InputStream execute(FileSystem fs) throws IOException {
523          int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096);
524          return fs.open(path, bufferSize);
525        }
526    
527      }
528    
529      /**
530       * Executor that performs a rename FileSystemAccess files system operation.
531       */
532      public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> {
533        private Path path;
534        private Path toPath;
535    
536        /**
537         * Creates a rename executor.
538         *
539         * @param path path to rename.
540         * @param toPath new name.
541         */
542        public FSRename(String path, String toPath) {
543          this.path = new Path(path);
544          this.toPath = new Path(toPath);
545        }
546    
547        /**
548         * Executes the filesystem operation.
549         *
550         * @param fs filesystem instance to use.
551         *
552         * @return <code>true</code> if the rename operation was successful,
553         *         <code>false</code> otherwise.
554         *
555         * @throws IOException thrown if an IO error occured.
556         */
557        @Override
558        public JSONObject execute(FileSystem fs) throws IOException {
559          boolean renamed = fs.rename(path, toPath);
560          return toJSON(HttpFSFileSystem.RENAME_JSON, renamed);
561        }
562    
563      }
564    
565      /**
566       * Executor that performs a set-owner FileSystemAccess files system operation.
567       */
568      public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> {
569        private Path path;
570        private String owner;
571        private String group;
572    
573        /**
574         * Creates a set-owner executor.
575         *
576         * @param path the path to set the owner.
577         * @param owner owner to set.
578         * @param group group to set.
579         */
580        public FSSetOwner(String path, String owner, String group) {
581          this.path = new Path(path);
582          this.owner = owner;
583          this.group = group;
584        }
585    
586        /**
587         * Executes the filesystem operation.
588         *
589         * @param fs filesystem instance to use.
590         *
591         * @return void.
592         *
593         * @throws IOException thrown if an IO error occured.
594         */
595        @Override
596        public Void execute(FileSystem fs) throws IOException {
597          fs.setOwner(path, owner, group);
598          return null;
599        }
600    
601      }
602    
603      /**
604       * Executor that performs a set-permission FileSystemAccess files system operation.
605       */
606      public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> {
607    
608        private Path path;
609        private String permission;
610    
611        /**
612         * Creates a set-permission executor.
613         *
614         * @param path path to set the permission.
615         * @param permission permission to set.
616         */
617        public FSSetPermission(String path, String permission) {
618          this.path = new Path(path);
619          this.permission = permission;
620        }
621    
622        /**
623         * Executes the filesystem operation.
624         *
625         * @param fs filesystem instance to use.
626         *
627         * @return void.
628         *
629         * @throws IOException thrown if an IO error occured.
630         */
631        @Override
632        public Void execute(FileSystem fs) throws IOException {
633          FsPermission fsPermission = getPermission(permission);
634          fs.setPermission(path, fsPermission);
635          return null;
636        }
637    
638      }
639    
640      /**
641       * Executor that performs a set-replication FileSystemAccess files system operation.
642       */
643      public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> {
644        private Path path;
645        private short replication;
646    
647        /**
648         * Creates a set-replication executor.
649         *
650         * @param path path to set the replication factor.
651         * @param replication replication factor to set.
652         */
653        public FSSetReplication(String path, short replication) {
654          this.path = new Path(path);
655          this.replication = replication;
656        }
657    
658        /**
659         * Executes the filesystem operation.
660         *
661         * @param fs filesystem instance to use.
662         *
663         * @return <code>true</code> if the replication value was set,
664         *         <code>false</code> otherwise.
665         *
666         * @throws IOException thrown if an IO error occured.
667         */
668        @Override
669        @SuppressWarnings("unchecked")
670        public JSONObject execute(FileSystem fs) throws IOException {
671          boolean ret = fs.setReplication(path, replication);
672          JSONObject json = new JSONObject();
673          json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret);
674          return json;
675        }
676    
677      }
678    
679      /**
680       * Executor that performs a set-times FileSystemAccess files system operation.
681       */
682      public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> {
683        private Path path;
684        private long mTime;
685        private long aTime;
686    
687        /**
688         * Creates a set-times executor.
689         *
690         * @param path path to set the times.
691         * @param mTime modified time to set.
692         * @param aTime access time to set.
693         */
694        public FSSetTimes(String path, long mTime, long aTime) {
695          this.path = new Path(path);
696          this.mTime = mTime;
697          this.aTime = aTime;
698        }
699    
700        /**
701         * Executes the filesystem operation.
702         *
703         * @param fs filesystem instance to use.
704         *
705         * @return void.
706         *
707         * @throws IOException thrown if an IO error occured.
708         */
709        @Override
710        public Void execute(FileSystem fs) throws IOException {
711          fs.setTimes(path, mTime, aTime);
712          return null;
713        }
714    
715      }
716    
717    }