001    /**
002    * Licensed to the Apache Software Foundation (ASF) under one
003    * or more contributor license agreements.  See the NOTICE file
004    * distributed with this work for additional information
005    * regarding copyright ownership.  The ASF licenses this file
006    * to you under the Apache License, Version 2.0 (the
007    * "License"); you may not use this file except in compliance
008    * with the License.  You may obtain a copy of the License at
009    *
010    *     http://www.apache.org/licenses/LICENSE-2.0
011    *
012    * Unless required by applicable law or agreed to in writing, software
013    * distributed under the License is distributed on an "AS IS" BASIS,
014    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015    * See the License for the specific language governing permissions and
016    * limitations under the License.
017    */
018    
019    package org.apache.hadoop.fs.http.server;
020    
021    import org.apache.hadoop.fs.ContentSummary;
022    import org.apache.hadoop.fs.FileChecksum;
023    import org.apache.hadoop.fs.FileStatus;
024    import org.apache.hadoop.fs.FileSystem;
025    import org.apache.hadoop.fs.GlobFilter;
026    import org.apache.hadoop.fs.Path;
027    import org.apache.hadoop.fs.PathFilter;
028    import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
029    import org.apache.hadoop.fs.permission.FsPermission;
030    import org.apache.hadoop.io.IOUtils;
031    import org.apache.hadoop.lib.service.FileSystemAccess;
032    import org.json.simple.JSONArray;
033    import org.json.simple.JSONObject;
034    
035    import java.io.IOException;
036    import java.io.InputStream;
037    import java.io.OutputStream;
038    import java.util.LinkedHashMap;
039    import java.util.Map;
040    
041    /**
042     * FileSystem operation executors used by {@link HttpFSServer}.
043     */
044    public class FSOperations {
045    
046      /**
047       * Converts a Unix permission octal & symbolic representation
048       * (i.e. 655 or -rwxr--r--) into a FileSystemAccess permission.
049       *
050       * @param str Unix permission symbolic representation.
051       *
052       * @return the FileSystemAccess permission. If the given string was
053       *         'default', it returns <code>FsPermission.getDefault()</code>.
054       */
055      private static FsPermission getPermission(String str) {
056        FsPermission permission;
057        if (str.equals(HttpFSFileSystem.DEFAULT_PERMISSION)) {
058          permission = FsPermission.getDefault();
059        } else if (str.length() == 3) {
060          permission = new FsPermission(Short.parseShort(str, 8));
061        } else {
062          permission = FsPermission.valueOf(str);
063        }
064        return permission;
065      }
066    
067      @SuppressWarnings({"unchecked", "deprecation"})
068      private static Map fileStatusToJSONRaw(FileStatus status, boolean emptyPathSuffix) {
069        Map json = new LinkedHashMap();
070        json.put(HttpFSFileSystem.PATH_SUFFIX_JSON, (emptyPathSuffix) ? "" : status.getPath().getName());
071        json.put(HttpFSFileSystem.TYPE_JSON, HttpFSFileSystem.FILE_TYPE.getType(status).toString());
072        json.put(HttpFSFileSystem.LENGTH_JSON, status.getLen());
073        json.put(HttpFSFileSystem.OWNER_JSON, status.getOwner());
074        json.put(HttpFSFileSystem.GROUP_JSON, status.getGroup());
075        json.put(HttpFSFileSystem.PERMISSION_JSON, HttpFSFileSystem.permissionToString(status.getPermission()));
076        json.put(HttpFSFileSystem.ACCESS_TIME_JSON, status.getAccessTime());
077        json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON, status.getModificationTime());
078        json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, status.getBlockSize());
079        json.put(HttpFSFileSystem.REPLICATION_JSON, status.getReplication());
080        return json;
081      }
082    
083      /**
084       * Converts a FileSystemAccess <code>FileStatus</code> object into a JSON
085       * object.
086       *
087       * @param status FileSystemAccess file status.
088       *
089       * @return The JSON representation of the file status.
090       */
091      @SuppressWarnings({"unchecked", "deprecation"})
092      private static Map fileStatusToJSON(FileStatus status) {
093        Map json = new LinkedHashMap();
094        json.put(HttpFSFileSystem.FILE_STATUS_JSON, fileStatusToJSONRaw(status, true));
095        return json;
096      }
097    
098      /**
099       * Converts a <code>FileChecksum</code> object into a JSON array
100       * object.
101       *
102       * @param checksum file checksum.
103       *
104       * @return The JSON representation of the file checksum.
105       */
106      @SuppressWarnings({"unchecked"})
107      private static Map fileChecksumToJSON(FileChecksum checksum) {
108        Map json = new LinkedHashMap();
109        json.put(HttpFSFileSystem.CHECKSUM_ALGORITHM_JSON, checksum.getAlgorithmName());
110        json.put(HttpFSFileSystem.CHECKSUM_BYTES_JSON,
111                 org.apache.hadoop.util.StringUtils.byteToHexString(checksum.getBytes()));
112        json.put(HttpFSFileSystem.CHECKSUM_LENGTH_JSON, checksum.getLength());
113        Map response = new LinkedHashMap();
114        response.put(HttpFSFileSystem.FILE_CHECKSUM_JSON, json);
115        return response;
116      }
117    
118      /**
119       * Converts a <code>ContentSummary</code> object into a JSON array
120       * object.
121       *
122       * @param contentSummary the content summary
123       *
124       * @return The JSON representation of the content summary.
125       */
126      @SuppressWarnings({"unchecked"})
127      private static Map contentSummaryToJSON(ContentSummary contentSummary) {
128        Map json = new LinkedHashMap();
129        json.put(HttpFSFileSystem.CONTENT_SUMMARY_DIRECTORY_COUNT_JSON, contentSummary.getDirectoryCount());
130        json.put(HttpFSFileSystem.CONTENT_SUMMARY_FILE_COUNT_JSON, contentSummary.getFileCount());
131        json.put(HttpFSFileSystem.CONTENT_SUMMARY_LENGTH_JSON, contentSummary.getLength());
132        json.put(HttpFSFileSystem.CONTENT_SUMMARY_QUOTA_JSON, contentSummary.getQuota());
133        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_CONSUMED_JSON, contentSummary.getSpaceConsumed());
134        json.put(HttpFSFileSystem.CONTENT_SUMMARY_SPACE_QUOTA_JSON, contentSummary.getSpaceQuota());
135        Map response = new LinkedHashMap();
136        response.put(HttpFSFileSystem.CONTENT_SUMMARY_JSON, json);
137        return response;
138      }
139    
140      /**
141       * Converts a FileSystemAccess <code>FileStatus</code> array into a JSON array
142       * object.
143       *
144       * @param status FileSystemAccess file status array.
145       * <code>SCHEME://HOST:PORT</code> in the file status.
146       *
147       * @return The JSON representation of the file status array.
148       */
149      @SuppressWarnings("unchecked")
150      private static Map fileStatusToJSON(FileStatus[] status) {
151        JSONArray json = new JSONArray();
152        if (status != null) {
153          for (FileStatus s : status) {
154            json.add(fileStatusToJSONRaw(s, false));
155          }
156        }
157        Map response = new LinkedHashMap();
158        Map temp = new LinkedHashMap();
159        temp.put(HttpFSFileSystem.FILE_STATUS_JSON, json);
160        response.put(HttpFSFileSystem.FILE_STATUSES_JSON, temp);
161        return response;
162      }
163    
164      /**
165       * Converts an object into a Json Map with with one key-value entry.
166       * <p/>
167       * It assumes the given value is either a JSON primitive type or a
168       * <code>JsonAware</code> instance.
169       *
170       * @param name name for the key of the entry.
171       * @param value for the value of the entry.
172       *
173       * @return the JSON representation of the key-value pair.
174       */
175      @SuppressWarnings("unchecked")
176      private static JSONObject toJSON(String name, Object value) {
177        JSONObject json = new JSONObject();
178        json.put(name, value);
179        return json;
180      }
181    
182      /**
183       * Executor that performs an append FileSystemAccess files system operation.
184       */
185      public static class FSAppend implements FileSystemAccess.FileSystemExecutor<Void> {
186        private InputStream is;
187        private Path path;
188    
189        /**
190         * Creates an Append executor.
191         *
192         * @param is input stream to append.
193         * @param path path of the file to append.
194         */
195        public FSAppend(InputStream is, String path) {
196          this.is = is;
197          this.path = new Path(path);
198        }
199    
200        /**
201         * Executes the filesystem operation.
202         *
203         * @param fs filesystem instance to use.
204         *
205         * @return void.
206         *
207         * @throws IOException thrown if an IO error occured.
208         */
209        @Override
210        public Void execute(FileSystem fs) throws IOException {
211          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
212          OutputStream os = fs.append(path, bufferSize);
213          IOUtils.copyBytes(is, os, bufferSize, true);
214          os.close();
215          return null;
216        }
217    
218      }
219    
220      /**
221       * Executor that performs a content-summary FileSystemAccess files system operation.
222       */
223      public static class FSContentSummary implements FileSystemAccess.FileSystemExecutor<Map> {
224        private Path path;
225    
226        /**
227         * Creates a content-summary executor.
228         *
229         * @param path the path to retrieve the content-summary.
230         */
231        public FSContentSummary(String path) {
232          this.path = new Path(path);
233        }
234    
235        /**
236         * Executes the filesystem operation.
237         *
238         * @param fs filesystem instance to use.
239         *
240         * @return a Map object (JSON friendly) with the content-summary.
241         *
242         * @throws IOException thrown if an IO error occured.
243         */
244        @Override
245        public Map execute(FileSystem fs) throws IOException {
246          ContentSummary contentSummary = fs.getContentSummary(path);
247          return contentSummaryToJSON(contentSummary);
248        }
249    
250      }
251    
252      /**
253       * Executor that performs a create FileSystemAccess files system operation.
254       */
255      public static class FSCreate implements FileSystemAccess.FileSystemExecutor<Void> {
256        private InputStream is;
257        private Path path;
258        private String permission;
259        private boolean override;
260        private short replication;
261        private long blockSize;
262    
263        /**
264         * Creates a Create executor.
265         *
266         * @param is input stream to for the file to create.
267         * @param path path of the file to create.
268         * @param perm permission for the file.
269         * @param override if the file should be overriden if it already exist.
270         * @param repl the replication factor for the file.
271         * @param blockSize the block size for the file.
272         */
273        public FSCreate(InputStream is, String path, String perm, boolean override, short repl, long blockSize) {
274          this.is = is;
275          this.path = new Path(path);
276          this.permission = perm;
277          this.override = override;
278          this.replication = repl;
279          this.blockSize = blockSize;
280        }
281    
282        /**
283         * Executes the filesystem operation.
284         *
285         * @param fs filesystem instance to use.
286         *
287         * @return The URI of the created file.
288         *
289         * @throws IOException thrown if an IO error occured.
290         */
291        @Override
292        public Void execute(FileSystem fs) throws IOException {
293          if (replication == -1) {
294            replication = fs.getDefaultReplication();
295          }
296          if (blockSize == -1) {
297            blockSize = fs.getDefaultBlockSize();
298          }
299          FsPermission fsPermission = getPermission(permission);
300          int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
301          OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
302          IOUtils.copyBytes(is, os, bufferSize, true);
303          os.close();
304          return null;
305        }
306    
307      }
308    
309      /**
310       * Executor that performs a delete FileSystemAccess files system operation.
311       */
312      public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> {
313        private Path path;
314        private boolean recursive;
315    
316        /**
317         * Creates a Delete executor.
318         *
319         * @param path path to delete.
320         * @param recursive if the delete should be recursive or not.
321         */
322        public FSDelete(String path, boolean recursive) {
323          this.path = new Path(path);
324          this.recursive = recursive;
325        }
326    
327        /**
328         * Executes the filesystem operation.
329         *
330         * @param fs filesystem instance to use.
331         *
332         * @return <code>true</code> if the delete operation was successful,
333         *         <code>false</code> otherwise.
334         *
335         * @throws IOException thrown if an IO error occured.
336         */
337        @Override
338        public JSONObject execute(FileSystem fs) throws IOException {
339          boolean deleted = fs.delete(path, recursive);
340          return toJSON(HttpFSFileSystem.DELETE_JSON.toLowerCase(), deleted);
341        }
342    
343      }
344    
345      /**
346       * Executor that performs a file-checksum FileSystemAccess files system operation.
347       */
348      public static class FSFileChecksum implements FileSystemAccess.FileSystemExecutor<Map> {
349        private Path path;
350    
351        /**
352         * Creates a file-checksum executor.
353         *
354         * @param path the path to retrieve the checksum.
355         */
356        public FSFileChecksum(String path) {
357          this.path = new Path(path);
358        }
359    
360        /**
361         * Executes the filesystem operation.
362         *
363         * @param fs filesystem instance to use.
364         *
365         * @return a Map object (JSON friendly) with the file checksum.
366         *
367         * @throws IOException thrown if an IO error occured.
368         */
369        @Override
370        public Map execute(FileSystem fs) throws IOException {
371          FileChecksum checksum = fs.getFileChecksum(path);
372          return fileChecksumToJSON(checksum);
373        }
374    
375      }
376    
377      /**
378       * Executor that performs a file-status FileSystemAccess files system operation.
379       */
380      public static class FSFileStatus implements FileSystemAccess.FileSystemExecutor<Map> {
381        private Path path;
382    
383        /**
384         * Creates a file-status executor.
385         *
386         * @param path the path to retrieve the status.
387         */
388        public FSFileStatus(String path) {
389          this.path = new Path(path);
390        }
391    
392        /**
393         * Executes the filesystem operation.
394         *
395         * @param fs filesystem instance to use.
396         *
397         * @return a Map object (JSON friendly) with the file status.
398         *
399         * @throws IOException thrown if an IO error occured.
400         */
401        @Override
402        public Map execute(FileSystem fs) throws IOException {
403          FileStatus status = fs.getFileStatus(path);
404          return fileStatusToJSON(status);
405        }
406    
407      }
408    
409      /**
410       * Executor that performs a home-dir FileSystemAccess files system operation.
411       */
412      public static class FSHomeDir implements FileSystemAccess.FileSystemExecutor<JSONObject> {
413    
414        /**
415         * Executes the filesystem operation.
416         *
417         * @param fs filesystem instance to use.
418         *
419         * @return a JSON object with the user home directory.
420         *
421         * @throws IOException thrown if an IO error occured.
422         */
423        @Override
424        @SuppressWarnings("unchecked")
425        public JSONObject execute(FileSystem fs) throws IOException {
426          Path homeDir = fs.getHomeDirectory();
427          JSONObject json = new JSONObject();
428          json.put(HttpFSFileSystem.HOME_DIR_JSON, homeDir.toUri().getPath());
429          return json;
430        }
431    
432      }
433    
434      /**
435       * Executor that performs a list-status FileSystemAccess files system operation.
436       */
437      public static class FSListStatus implements FileSystemAccess.FileSystemExecutor<Map>, PathFilter {
438        private Path path;
439        private PathFilter filter;
440    
441        /**
442         * Creates a list-status executor.
443         *
444         * @param path the directory to retrieve the status of its contents.
445         * @param filter glob filter to use.
446         *
447         * @throws IOException thrown if the filter expression is incorrect.
448         */
449        public FSListStatus(String path, String filter) throws IOException {
450          this.path = new Path(path);
451          this.filter = (filter == null) ? this : new GlobFilter(filter);
452        }
453    
454        /**
455         * Executes the filesystem operation.
456         *
457         * @param fs filesystem instance to use.
458         *
459         * @return a Map with the file status of the directory
460         *         contents.
461         *
462         * @throws IOException thrown if an IO error occured.
463         */
464        @Override
465        public Map execute(FileSystem fs) throws IOException {
466          FileStatus[] status = fs.listStatus(path, filter);
467          return fileStatusToJSON(status);
468        }
469    
470        @Override
471        public boolean accept(Path path) {
472          return true;
473        }
474    
475      }
476    
477      /**
478       * Executor that performs a mkdirs FileSystemAccess files system operation.
479       */
480      public static class FSMkdirs implements FileSystemAccess.FileSystemExecutor<JSONObject> {
481    
482        private Path path;
483        private String permission;
484    
485        /**
486         * Creates a mkdirs executor.
487         *
488         * @param path directory path to create.
489         * @param permission permission to use.
490         */
491        public FSMkdirs(String path, String permission) {
492          this.path = new Path(path);
493          this.permission = permission;
494        }
495    
496        /**
497         * Executes the filesystem operation.
498         *
499         * @param fs filesystem instance to use.
500         *
501         * @return <code>true</code> if the mkdirs operation was successful,
502         *         <code>false</code> otherwise.
503         *
504         * @throws IOException thrown if an IO error occured.
505         */
506        @Override
507        public JSONObject execute(FileSystem fs) throws IOException {
508          FsPermission fsPermission = getPermission(permission);
509          boolean mkdirs = fs.mkdirs(path, fsPermission);
510          return toJSON(HttpFSFileSystem.MKDIRS_JSON, mkdirs);
511        }
512    
513      }
514    
515      /**
516       * Executor that performs a open FileSystemAccess files system operation.
517       */
518      public static class FSOpen implements FileSystemAccess.FileSystemExecutor<InputStream> {
519        private Path path;
520    
521        /**
522         * Creates a open executor.
523         *
524         * @param path file to open.
525         */
526        public FSOpen(String path) {
527          this.path = new Path(path);
528        }
529    
530        /**
531         * Executes the filesystem operation.
532         *
533         * @param fs filesystem instance to use.
534         *
535         * @return The inputstream of the file.
536         *
537         * @throws IOException thrown if an IO error occured.
538         */
539        @Override
540        public InputStream execute(FileSystem fs) throws IOException {
541          int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096);
542          return fs.open(path, bufferSize);
543        }
544    
545      }
546    
547      /**
548       * Executor that performs a rename FileSystemAccess files system operation.
549       */
550      public static class FSRename implements FileSystemAccess.FileSystemExecutor<JSONObject> {
551        private Path path;
552        private Path toPath;
553    
554        /**
555         * Creates a rename executor.
556         *
557         * @param path path to rename.
558         * @param toPath new name.
559         */
560        public FSRename(String path, String toPath) {
561          this.path = new Path(path);
562          this.toPath = new Path(toPath);
563        }
564    
565        /**
566         * Executes the filesystem operation.
567         *
568         * @param fs filesystem instance to use.
569         *
570         * @return <code>true</code> if the rename operation was successful,
571         *         <code>false</code> otherwise.
572         *
573         * @throws IOException thrown if an IO error occured.
574         */
575        @Override
576        public JSONObject execute(FileSystem fs) throws IOException {
577          boolean renamed = fs.rename(path, toPath);
578          return toJSON(HttpFSFileSystem.RENAME_JSON, renamed);
579        }
580    
581      }
582    
583      /**
584       * Executor that performs a set-owner FileSystemAccess files system operation.
585       */
586      public static class FSSetOwner implements FileSystemAccess.FileSystemExecutor<Void> {
587        private Path path;
588        private String owner;
589        private String group;
590    
591        /**
592         * Creates a set-owner executor.
593         *
594         * @param path the path to set the owner.
595         * @param owner owner to set.
596         * @param group group to set.
597         */
598        public FSSetOwner(String path, String owner, String group) {
599          this.path = new Path(path);
600          this.owner = owner;
601          this.group = group;
602        }
603    
604        /**
605         * Executes the filesystem operation.
606         *
607         * @param fs filesystem instance to use.
608         *
609         * @return void.
610         *
611         * @throws IOException thrown if an IO error occured.
612         */
613        @Override
614        public Void execute(FileSystem fs) throws IOException {
615          fs.setOwner(path, owner, group);
616          return null;
617        }
618    
619      }
620    
621      /**
622       * Executor that performs a set-permission FileSystemAccess files system operation.
623       */
624      public static class FSSetPermission implements FileSystemAccess.FileSystemExecutor<Void> {
625    
626        private Path path;
627        private String permission;
628    
629        /**
630         * Creates a set-permission executor.
631         *
632         * @param path path to set the permission.
633         * @param permission permission to set.
634         */
635        public FSSetPermission(String path, String permission) {
636          this.path = new Path(path);
637          this.permission = permission;
638        }
639    
640        /**
641         * Executes the filesystem operation.
642         *
643         * @param fs filesystem instance to use.
644         *
645         * @return void.
646         *
647         * @throws IOException thrown if an IO error occured.
648         */
649        @Override
650        public Void execute(FileSystem fs) throws IOException {
651          FsPermission fsPermission = getPermission(permission);
652          fs.setPermission(path, fsPermission);
653          return null;
654        }
655    
656      }
657    
658      /**
659       * Executor that performs a set-replication FileSystemAccess files system operation.
660       */
661      public static class FSSetReplication implements FileSystemAccess.FileSystemExecutor<JSONObject> {
662        private Path path;
663        private short replication;
664    
665        /**
666         * Creates a set-replication executor.
667         *
668         * @param path path to set the replication factor.
669         * @param replication replication factor to set.
670         */
671        public FSSetReplication(String path, short replication) {
672          this.path = new Path(path);
673          this.replication = replication;
674        }
675    
676        /**
677         * Executes the filesystem operation.
678         *
679         * @param fs filesystem instance to use.
680         *
681         * @return <code>true</code> if the replication value was set,
682         *         <code>false</code> otherwise.
683         *
684         * @throws IOException thrown if an IO error occured.
685         */
686        @Override
687        @SuppressWarnings("unchecked")
688        public JSONObject execute(FileSystem fs) throws IOException {
689          boolean ret = fs.setReplication(path, replication);
690          JSONObject json = new JSONObject();
691          json.put(HttpFSFileSystem.SET_REPLICATION_JSON, ret);
692          return json;
693        }
694    
695      }
696    
697      /**
698       * Executor that performs a set-times FileSystemAccess files system operation.
699       */
700      public static class FSSetTimes implements FileSystemAccess.FileSystemExecutor<Void> {
701        private Path path;
702        private long mTime;
703        private long aTime;
704    
705        /**
706         * Creates a set-times executor.
707         *
708         * @param path path to set the times.
709         * @param mTime modified time to set.
710         * @param aTime access time to set.
711         */
712        public FSSetTimes(String path, long mTime, long aTime) {
713          this.path = new Path(path);
714          this.mTime = mTime;
715          this.aTime = aTime;
716        }
717    
718        /**
719         * Executes the filesystem operation.
720         *
721         * @param fs filesystem instance to use.
722         *
723         * @return void.
724         *
725         * @throws IOException thrown if an IO error occured.
726         */
727        @Override
728        public Void execute(FileSystem fs) throws IOException {
729          fs.setTimes(path, mTime, aTime);
730          return null;
731        }
732    
733      }
734    
735    }