001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs.http.client;
019    
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.hadoop.fs.ContentSummary;
022    import org.apache.hadoop.fs.FSDataInputStream;
023    import org.apache.hadoop.fs.FSDataOutputStream;
024    import org.apache.hadoop.fs.FileChecksum;
025    import org.apache.hadoop.fs.FileStatus;
026    import org.apache.hadoop.fs.FileSystem;
027    import org.apache.hadoop.fs.Path;
028    import org.apache.hadoop.fs.PositionedReadable;
029    import org.apache.hadoop.fs.Seekable;
030    import org.apache.hadoop.fs.permission.FsPermission;
031    import org.apache.hadoop.security.UserGroupInformation;
032    import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
033    import org.apache.hadoop.security.authentication.client.Authenticator;
034    import org.apache.hadoop.util.Progressable;
035    import org.apache.hadoop.util.ReflectionUtils;
036    import org.apache.hadoop.util.StringUtils;
037    import org.json.simple.JSONArray;
038    import org.json.simple.JSONObject;
039    import org.json.simple.parser.JSONParser;
040    import org.json.simple.parser.ParseException;
041    
042    import java.io.BufferedInputStream;
043    import java.io.BufferedOutputStream;
044    import java.io.DataInput;
045    import java.io.DataOutput;
046    import java.io.FileNotFoundException;
047    import java.io.FilterInputStream;
048    import java.io.IOException;
049    import java.io.InputStream;
050    import java.io.InputStreamReader;
051    import java.io.OutputStream;
052    import java.lang.reflect.Constructor;
053    import java.net.HttpURLConnection;
054    import java.net.URI;
055    import java.net.URISyntaxException;
056    import java.net.URL;
057    import java.net.URLEncoder;
058    import java.text.MessageFormat;
059    import java.util.HashMap;
060    import java.util.Map;
061    
062    /**
063     * HttpFSServer implementation of the FileSystemAccess FileSystem.
064     * <p/>
065     * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server.
066     */
067    public class HttpFSFileSystem extends FileSystem {
068    
069      public static final String SERVICE_NAME = "/webhdfs";
070    
071      public static final String SERVICE_VERSION = "/v1";
072    
073      public static final String SERVICE_PREFIX = SERVICE_NAME + SERVICE_VERSION;
074    
075      public static final String OP_PARAM = "op";
076      public static final String DO_AS_PARAM = "doas";
077      public static final String OVERWRITE_PARAM = "overwrite";
078      public static final String REPLICATION_PARAM = "replication";
079      public static final String BLOCKSIZE_PARAM = "blocksize";
080      public static final String PERMISSION_PARAM = "permission";
081      public static final String DESTINATION_PARAM = "destination";
082      public static final String RECURSIVE_PARAM = "recursive";
083      public static final String OWNER_PARAM = "owner";
084      public static final String GROUP_PARAM = "group";
085      public static final String MODIFICATION_TIME_PARAM = "modificationtime";
086      public static final String ACCESS_TIME_PARAM = "accesstime";
087      public static final String RENEWER_PARAM = "renewer";
088    
089      public static final String DEFAULT_PERMISSION = "default";
090    
091      public static final String RENAME_JSON = "boolean";
092    
093      public static final String DELETE_JSON = "boolean";
094    
095      public static final String MKDIRS_JSON = "boolean";
096    
097      public static final String HOME_DIR_JSON = "Path";
098    
099      public static final String SET_REPLICATION_JSON = "boolean";
100    
101      public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream";
102    
103      public static enum FILE_TYPE {
104        FILE, DIRECTORY, SYMLINK;
105    
106        public static FILE_TYPE getType(FileStatus fileStatus) {
107          if (fileStatus.isFile()) {
108            return FILE;
109          }
110          if (fileStatus.isDirectory()) {
111            return DIRECTORY;
112          }
113          if (fileStatus.isSymlink()) {
114            return SYMLINK;
115          }
116          throw new IllegalArgumentException("Could not determine filetype for: " +
117                                             fileStatus.getPath());
118        }
119      }
120    
121      public static final String FILE_STATUSES_JSON = "FileStatuses";
122      public static final String FILE_STATUS_JSON = "FileStatus";
123      public static final String PATH_SUFFIX_JSON = "pathSuffix";
124      public static final String TYPE_JSON = "type";
125      public static final String LENGTH_JSON = "length";
126      public static final String OWNER_JSON = "owner";
127      public static final String GROUP_JSON = "group";
128      public static final String PERMISSION_JSON = "permission";
129      public static final String ACCESS_TIME_JSON = "accessTime";
130      public static final String MODIFICATION_TIME_JSON = "modificationTime";
131      public static final String BLOCK_SIZE_JSON = "blockSize";
132      public static final String REPLICATION_JSON = "replication";
133    
134      public static final String FILE_CHECKSUM_JSON = "FileChecksum";
135      public static final String CHECKSUM_ALGORITHM_JSON = "algorithm";
136      public static final String CHECKSUM_BYTES_JSON = "bytes";
137      public static final String CHECKSUM_LENGTH_JSON = "length";
138    
139      public static final String CONTENT_SUMMARY_JSON = "ContentSummary";
140      public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount";
141      public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount";
142      public static final String CONTENT_SUMMARY_LENGTH_JSON = "length";
143      public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota";
144      public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed";
145      public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota";
146    
147      public static final String DELEGATION_TOKEN_JSON = "Token";
148      public static final String DELEGATION_TOKEN_URL_STRING_JSON = "urlString";
149    
150      public static final String ERROR_JSON = "RemoteException";
151      public static final String ERROR_EXCEPTION_JSON = "exception";
152      public static final String ERROR_CLASSNAME_JSON = "javaClassName";
153      public static final String ERROR_MESSAGE_JSON = "message";
154    
155      public static final int HTTP_TEMPORARY_REDIRECT = 307;
156    
157    
158      /**
159       * Get operations.
160       */
161      public enum GetOpValues {
162        OPEN, GETFILESTATUS, LISTSTATUS, GETHOMEDIRECTORY, GETCONTENTSUMMARY, GETFILECHECKSUM,
163        GETDELEGATIONTOKEN, GETFILEBLOCKLOCATIONS, INSTRUMENTATION
164      }
165    
166      /**
167       * Post operations.
168       */
169      public static enum PostOpValues {
170        APPEND
171      }
172    
173      /**
174       * Put operations.
175       */
176      public static enum PutOpValues {
177        CREATE, MKDIRS, RENAME, SETOWNER, SETPERMISSION, SETREPLICATION, SETTIMES,
178        RENEWDELEGATIONTOKEN, CANCELDELEGATIONTOKEN
179      }
180    
181      /**
182       * Delete operations.
183       */
184      public static enum DeleteOpValues {
185        DELETE
186      }
187    
188      private static final String HTTP_GET = "GET";
189      private static final String HTTP_PUT = "PUT";
190      private static final String HTTP_POST = "POST";
191      private static final String HTTP_DELETE = "DELETE";
192    
193      private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
194      private URI uri;
195      private Path workingDir;
196      private String doAs;
197    
198      /**
199       * Convenience method that creates a <code>HttpURLConnection</code> for the
200       * HttpFSServer file system operations.
201       * <p/>
202       * This methods performs and injects any needed authentication credentials
203       * via the {@link #getConnection(URL, String)} method
204       *
205       * @param method the HTTP method.
206       * @param params the query string parameters.
207       * @param path the file path
208       * @param makeQualified if the path should be 'makeQualified'
209       *
210       * @return a <code>HttpURLConnection</code> for the HttpFSServer server,
211       *         authenticated and ready to use for the specified path and file system operation.
212       *
213       * @throws IOException thrown if an IO error occurrs.
214       */
215      private HttpURLConnection getConnection(String method, Map<String, String> params,
216                                              Path path, boolean makeQualified) throws IOException {
217        params.put(DO_AS_PARAM, doAs);
218        if (makeQualified) {
219          path = makeQualified(path);
220        }
221        URI uri = path.toUri();
222        StringBuilder sb = new StringBuilder();
223        sb.append(uri.getScheme()).append("://").append(uri.getAuthority()).
224          append(SERVICE_PREFIX).append(uri.getPath());
225    
226        String separator = "?";
227        for (Map.Entry<String, String> entry : params.entrySet()) {
228          sb.append(separator).append(entry.getKey()).append("=").
229            append(URLEncoder.encode(entry.getValue(), "UTF8"));
230          separator = "&";
231        }
232        URL url = new URL(sb.toString());
233        return getConnection(url, method);
234      }
235    
236      /**
237       * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL.
238       * <p/>
239       * This methods performs and injects any needed authentication credentials.
240       *
241       * @param url url to connect to.
242       * @param method the HTTP method.
243       *
244       * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for
245       *         the specified path and file system operation.
246       *
247       * @throws IOException thrown if an IO error occurrs.
248       */
249      private HttpURLConnection getConnection(URL url, String method) throws IOException {
250        Class<? extends Authenticator> klass =
251          getConf().getClass("httpfs.authenticator.class", HttpKerberosAuthenticator.class, Authenticator.class);
252        Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf());
253        try {
254          HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken);
255          conn.setRequestMethod(method);
256          if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) {
257            conn.setDoOutput(true);
258          }
259          return conn;
260        } catch (Exception ex) {
261          throw new IOException(ex);
262        }
263      }
264    
265      /**
266       * Convenience method that JSON Parses the <code>InputStream</code> of a <code>HttpURLConnection</code>.
267       *
268       * @param conn the <code>HttpURLConnection</code>.
269       *
270       * @return the parsed JSON object.
271       *
272       * @throws IOException thrown if the <code>InputStream</code> could not be JSON parsed.
273       */
274      private static Object jsonParse(HttpURLConnection conn) throws IOException {
275        try {
276          JSONParser parser = new JSONParser();
277          return parser.parse(new InputStreamReader(conn.getInputStream()));
278        } catch (ParseException ex) {
279          throw new IOException("JSON parser error, " + ex.getMessage(), ex);
280        }
281      }
282    
283      /**
284       * Validates the status of an <code>HttpURLConnection</code> against an expected HTTP
285       * status code. If the current status code is not the expected one it throws an exception
286       * with a detail message using Server side error messages if available.
287       *
288       * @param conn the <code>HttpURLConnection</code>.
289       * @param expected the expected HTTP status code.
290       *
291       * @throws IOException thrown if the current status code does not match the expected one.
292       */
293      private static void validateResponse(HttpURLConnection conn, int expected) throws IOException {
294        int status = conn.getResponseCode();
295        if (status != expected) {
296          try {
297            JSONObject json = (JSONObject) jsonParse(conn);
298            json = (JSONObject) json.get(ERROR_JSON);
299            String message = (String) json.get(ERROR_MESSAGE_JSON);
300            String exception = (String) json.get(ERROR_EXCEPTION_JSON);
301            String className = (String) json.get(ERROR_CLASSNAME_JSON);
302    
303            try {
304              ClassLoader cl = HttpFSFileSystem.class.getClassLoader();
305              Class klass = cl.loadClass(className);
306              Constructor constr = klass.getConstructor(String.class);
307              throw (IOException) constr.newInstance(message);
308            } catch (IOException ex) {
309              throw ex;
310            } catch (Exception ex) {
311              throw new IOException(MessageFormat.format("{0} - {1}", exception, message));
312            }
313          } catch (IOException ex) {
314            if (ex.getCause() instanceof IOException) {
315              throw (IOException) ex.getCause();
316            }
317            throw new IOException(MessageFormat.format("HTTP status [{0}], {1}", status, conn.getResponseMessage()));
318          }
319        }
320      }
321    
322      /**
323       * Called after a new FileSystem instance is constructed.
324       *
325       * @param name a uri whose authority section names the host, port, etc. for this FileSystem
326       * @param conf the configuration
327       */
328      @Override
329      public void initialize(URI name, Configuration conf) throws IOException {
330        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
331        doAs = ugi.getUserName();
332        super.initialize(name, conf);
333        try {
334          uri = new URI(name.getScheme() + "://" + name.getHost() + ":" + name.getPort());
335        } catch (URISyntaxException ex) {
336          throw new IOException(ex);
337        }
338      }
339    
340      /**
341       * Returns a URI whose scheme and authority identify this FileSystem.
342       *
343       * @return the URI whose scheme and authority identify this FileSystem.
344       */
345      @Override
346      public URI getUri() {
347        return uri;
348      }
349    
350      /**
351       * HttpFSServer subclass of the <code>FSDataInputStream</code>.
352       * <p/>
353       * This implementation does not support the
354       * <code>PositionReadable</code> and <code>Seekable</code> methods.
355       */
356      private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable {
357    
358        protected HttpFSDataInputStream(InputStream in, int bufferSize) {
359          super(new BufferedInputStream(in, bufferSize));
360        }
361    
362        @Override
363        public int read(long position, byte[] buffer, int offset, int length) throws IOException {
364          throw new UnsupportedOperationException();
365        }
366    
367        @Override
368        public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
369          throw new UnsupportedOperationException();
370        }
371    
372        @Override
373        public void readFully(long position, byte[] buffer) throws IOException {
374          throw new UnsupportedOperationException();
375        }
376    
377        @Override
378        public void seek(long pos) throws IOException {
379          throw new UnsupportedOperationException();
380        }
381    
382        @Override
383        public long getPos() throws IOException {
384          throw new UnsupportedOperationException();
385        }
386    
387        @Override
388        public boolean seekToNewSource(long targetPos) throws IOException {
389          throw new UnsupportedOperationException();
390        }
391      }
392    
393      /**
394       * Opens an FSDataInputStream at the indicated Path.
395       * </p>
396       * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the
397       * <code>PositionReadable</code> and <code>Seekable</code> methods.
398       *
399       * @param f the file name to open
400       * @param bufferSize the size of the buffer to be used.
401       */
402      @Override
403      public FSDataInputStream open(Path f, int bufferSize) throws IOException {
404        Map<String, String> params = new HashMap<String, String>();
405        params.put(OP_PARAM, GetOpValues.OPEN.toString());
406        HttpURLConnection conn = getConnection(HTTP_GET, params, f, true);
407        validateResponse(conn, HttpURLConnection.HTTP_OK);
408        return new FSDataInputStream(new HttpFSDataInputStream(conn.getInputStream(), bufferSize));
409      }
410    
411      /**
412       * HttpFSServer subclass of the <code>FSDataOutputStream</code>.
413       * <p/>
414       * This implementation closes the underlying HTTP connection validating the Http connection status
415       * at closing time.
416       */
417      private static class HttpFSDataOutputStream extends FSDataOutputStream {
418        private HttpURLConnection conn;
419        private int closeStatus;
420    
421        public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats)
422          throws IOException {
423          super(out, stats);
424          this.conn = conn;
425          this.closeStatus = closeStatus;
426        }
427    
428        @Override
429        public void close() throws IOException {
430          try {
431            super.close();
432          } finally {
433            validateResponse(conn, closeStatus);
434          }
435        }
436    
437      }
438    
439      /**
440       * Converts a <code>FsPermission</code> to a Unix octal representation.
441       *
442       * @param p the permission.
443       *
444       * @return the Unix string symbolic reprentation.
445       */
446      public static String permissionToString(FsPermission p) {
447        return (p == null) ? DEFAULT_PERMISSION : Integer.toString(p.toShort(), 8);
448      }
449    
450      /*
451       * Common handling for uploading data for create and append operations.
452       */
453      private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params,
454                                            int bufferSize, int expectedStatus) throws IOException {
455        HttpURLConnection conn = getConnection(method, params, f, true);
456        conn.setInstanceFollowRedirects(false);
457        boolean exceptionAlreadyHandled = false;
458        try {
459          if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) {
460            exceptionAlreadyHandled = true;
461            String location = conn.getHeaderField("Location");
462            if (location != null) {
463              conn = getConnection(new URL(location), method);
464              conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE);
465              try {
466                OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize);
467                return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics);
468              } catch (IOException ex) {
469                validateResponse(conn, expectedStatus);
470                throw ex;
471              }
472            } else {
473              validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
474              throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]");
475            }
476          } else {
477            throw new IOException(
478              MessageFormat.format("Expected HTTP status was [307], received [{0}]",
479                                   conn.getResponseCode()));
480          }
481        } catch (IOException ex) {
482          if (exceptionAlreadyHandled) {
483            throw ex;
484          } else {
485            validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
486            throw ex;
487          }
488        }
489      }
490    
491    
492      /**
493       * Opens an FSDataOutputStream at the indicated Path with write-progress
494       * reporting.
495       * <p/>
496       * IMPORTANT: The <code>Progressable</code> parameter is not used.
497       *
498       * @param f the file name to open.
499       * @param permission file permission.
500       * @param overwrite if a file with this name already exists, then if true,
501       * the file will be overwritten, and if false an error will be thrown.
502       * @param bufferSize the size of the buffer to be used.
503       * @param replication required block replication for the file.
504       * @param blockSize block size.
505       * @param progress progressable.
506       *
507       * @throws IOException
508       * @see #setPermission(Path, FsPermission)
509       */
510      @Override
511      public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize,
512                                       short replication, long blockSize, Progressable progress) throws IOException {
513        Map<String, String> params = new HashMap<String, String>();
514        params.put(OP_PARAM, PutOpValues.CREATE.toString());
515        params.put(OVERWRITE_PARAM, Boolean.toString(overwrite));
516        params.put(REPLICATION_PARAM, Short.toString(replication));
517        params.put(BLOCKSIZE_PARAM, Long.toString(blockSize));
518        params.put(PERMISSION_PARAM, permissionToString(permission));
519        return uploadData(HTTP_PUT, f, params, bufferSize, HttpURLConnection.HTTP_CREATED);
520      }
521    
522    
523      /**
524       * Append to an existing file (optional operation).
525       * <p/>
526       * IMPORTANT: The <code>Progressable</code> parameter is not used.
527       *
528       * @param f the existing file to be appended.
529       * @param bufferSize the size of the buffer to be used.
530       * @param progress for reporting progress if it is not null.
531       *
532       * @throws IOException
533       */
534      @Override
535      public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException {
536        Map<String, String> params = new HashMap<String, String>();
537        params.put(OP_PARAM, PostOpValues.APPEND.toString());
538        return uploadData(HTTP_POST, f, params, bufferSize, HttpURLConnection.HTTP_OK);
539      }
540    
541      /**
542       * Renames Path src to Path dst.  Can take place on local fs
543       * or remote DFS.
544       */
545      @Override
546      public boolean rename(Path src, Path dst) throws IOException {
547        Map<String, String> params = new HashMap<String, String>();
548        params.put(OP_PARAM, PutOpValues.RENAME.toString());
549        params.put(DESTINATION_PARAM, dst.toString());
550        HttpURLConnection conn = getConnection(HTTP_PUT, params, src, true);
551        validateResponse(conn, HttpURLConnection.HTTP_OK);
552        JSONObject json = (JSONObject) jsonParse(conn);
553        return (Boolean) json.get(RENAME_JSON);
554      }
555    
556      /**
557       * Delete a file.
558       *
559       * @deprecated Use delete(Path, boolean) instead
560       */
561      @SuppressWarnings({"deprecation"})
562      @Deprecated
563      @Override
564      public boolean delete(Path f) throws IOException {
565        return delete(f, false);
566      }
567    
568      /**
569       * Delete a file.
570       *
571       * @param f the path to delete.
572       * @param recursive if path is a directory and set to
573       * true, the directory is deleted else throws an exception. In
574       * case of a file the recursive can be set to either true or false.
575       *
576       * @return true if delete is successful else false.
577       *
578       * @throws IOException
579       */
580      @Override
581      public boolean delete(Path f, boolean recursive) throws IOException {
582        Map<String, String> params = new HashMap<String, String>();
583        params.put(OP_PARAM, DeleteOpValues.DELETE.toString());
584        params.put(RECURSIVE_PARAM, Boolean.toString(recursive));
585        HttpURLConnection conn = getConnection(HTTP_DELETE, params, f, true);
586        validateResponse(conn, HttpURLConnection.HTTP_OK);
587        JSONObject json = (JSONObject) jsonParse(conn);
588        return (Boolean) json.get(DELETE_JSON);
589      }
590    
591      /**
592       * List the statuses of the files/directories in the given path if the path is
593       * a directory.
594       *
595       * @param f given path
596       *
597       * @return the statuses of the files/directories in the given patch
598       *
599       * @throws IOException
600       */
601      @Override
602      public FileStatus[] listStatus(Path f) throws IOException {
603        Map<String, String> params = new HashMap<String, String>();
604        params.put(OP_PARAM, GetOpValues.LISTSTATUS.toString());
605        HttpURLConnection conn = getConnection(HTTP_GET, params, f, true);
606        validateResponse(conn, HttpURLConnection.HTTP_OK);
607        JSONObject json = (JSONObject) jsonParse(conn);
608        json = (JSONObject) json.get(FILE_STATUSES_JSON);
609        JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON);
610        FileStatus[] array = new FileStatus[jsonArray.size()];
611        f = makeQualified(f);
612        for (int i = 0; i < jsonArray.size(); i++) {
613          array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i));
614        }
615        return array;
616      }
617    
618      /**
619       * Set the current working directory for the given file system. All relative
620       * paths will be resolved relative to it.
621       *
622       * @param newDir new directory.
623       */
624      @Override
625      public void setWorkingDirectory(Path newDir) {
626        workingDir = newDir;
627      }
628    
629      /**
630       * Get the current working directory for the given file system
631       *
632       * @return the directory pathname
633       */
634      @Override
635      public Path getWorkingDirectory() {
636        if (workingDir == null) {
637          workingDir = getHomeDirectory();
638        }
639        return workingDir;
640      }
641    
642      /**
643       * Make the given file and all non-existent parents into
644       * directories. Has the semantics of Unix 'mkdir -p'.
645       * Existence of the directory hierarchy is not an error.
646       */
647      @Override
648      public boolean mkdirs(Path f, FsPermission permission) throws IOException {
649        Map<String, String> params = new HashMap<String, String>();
650        params.put(OP_PARAM, PutOpValues.MKDIRS.toString());
651        params.put(PERMISSION_PARAM, permissionToString(permission));
652        HttpURLConnection conn = getConnection(HTTP_PUT, params, f, true);
653        validateResponse(conn, HttpURLConnection.HTTP_OK);
654        JSONObject json = (JSONObject) jsonParse(conn);
655        return (Boolean) json.get(MKDIRS_JSON);
656      }
657    
658      /**
659       * Return a file status object that represents the path.
660       *
661       * @param f The path we want information from
662       *
663       * @return a FileStatus object
664       *
665       * @throws FileNotFoundException when the path does not exist;
666       * IOException see specific implementation
667       */
668      @Override
669      public FileStatus getFileStatus(Path f) throws IOException {
670        Map<String, String> params = new HashMap<String, String>();
671        params.put(OP_PARAM, GetOpValues.GETFILESTATUS.toString());
672        HttpURLConnection conn = getConnection(HTTP_GET, params, f, true);
673        validateResponse(conn, HttpURLConnection.HTTP_OK);
674        JSONObject json = (JSONObject) jsonParse(conn);
675        json = (JSONObject) json.get(FILE_STATUS_JSON);
676        f = makeQualified(f);
677        return createFileStatus(f, json);
678      }
679    
680      /**
681       * Return the current user's home directory in this filesystem.
682       * The default implementation returns "/user/$USER/".
683       */
684      @Override
685      public Path getHomeDirectory() {
686        Map<String, String> params = new HashMap<String, String>();
687        params.put(OP_PARAM, GetOpValues.GETHOMEDIRECTORY.toString());
688        try {
689          HttpURLConnection conn = getConnection(HTTP_GET, params, new Path(getUri().toString(), "/"), false);
690          validateResponse(conn, HttpURLConnection.HTTP_OK);
691          JSONObject json = (JSONObject) jsonParse(conn);
692          return new Path((String) json.get(HOME_DIR_JSON));
693        } catch (IOException ex) {
694          throw new RuntimeException(ex);
695        }
696      }
697    
698      /**
699       * Set owner of a path (i.e. a file or a directory).
700       * The parameters username and groupname cannot both be null.
701       *
702       * @param p The path
703       * @param username If it is null, the original username remains unchanged.
704       * @param groupname If it is null, the original groupname remains unchanged.
705       */
706      @Override
707      public void setOwner(Path p, String username, String groupname) throws IOException {
708        Map<String, String> params = new HashMap<String, String>();
709        params.put(OP_PARAM, PutOpValues.SETOWNER.toString());
710        params.put(OWNER_PARAM, username);
711        params.put(GROUP_PARAM, groupname);
712        HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true);
713        validateResponse(conn, HttpURLConnection.HTTP_OK);
714      }
715    
716      /**
717       * Set permission of a path.
718       *
719       * @param p path.
720       * @param permission permission.
721       */
722      @Override
723      public void setPermission(Path p, FsPermission permission) throws IOException {
724        Map<String, String> params = new HashMap<String, String>();
725        params.put(OP_PARAM, PutOpValues.SETPERMISSION.toString());
726        params.put(PERMISSION_PARAM, permissionToString(permission));
727        HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true);
728        validateResponse(conn, HttpURLConnection.HTTP_OK);
729      }
730    
731      /**
732       * Set access time of a file
733       *
734       * @param p The path
735       * @param mtime Set the modification time of this file.
736       * The number of milliseconds since Jan 1, 1970.
737       * A value of -1 means that this call should not set modification time.
738       * @param atime Set the access time of this file.
739       * The number of milliseconds since Jan 1, 1970.
740       * A value of -1 means that this call should not set access time.
741       */
742      @Override
743      public void setTimes(Path p, long mtime, long atime) throws IOException {
744        Map<String, String> params = new HashMap<String, String>();
745        params.put(OP_PARAM, PutOpValues.SETTIMES.toString());
746        params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime));
747        params.put(ACCESS_TIME_PARAM, Long.toString(atime));
748        HttpURLConnection conn = getConnection(HTTP_PUT, params, p, true);
749        validateResponse(conn, HttpURLConnection.HTTP_OK);
750      }
751    
752      /**
753       * Set replication for an existing file.
754       *
755       * @param src file name
756       * @param replication new replication
757       *
758       * @return true if successful;
759       *         false if file does not exist or is a directory
760       *
761       * @throws IOException
762       */
763      @Override
764      public boolean setReplication(Path src, short replication) throws IOException {
765        Map<String, String> params = new HashMap<String, String>();
766        params.put(OP_PARAM, PutOpValues.SETREPLICATION.toString());
767        params.put(REPLICATION_PARAM, Short.toString(replication));
768        HttpURLConnection conn = getConnection(HTTP_PUT, params, src, true);
769        validateResponse(conn, HttpURLConnection.HTTP_OK);
770        JSONObject json = (JSONObject) jsonParse(conn);
771        return (Boolean) json.get(SET_REPLICATION_JSON);
772      }
773    
774      /**
775       * Creates a <code>FileStatus</code> object using a JSON file-status payload
776       * received from a HttpFSServer server.
777       *
778       * @param json a JSON file-status payload received from a HttpFSServer server
779       *
780       * @return the corresponding <code>FileStatus</code>
781       */
782      private FileStatus createFileStatus(Path parent, JSONObject json) {
783        String pathSuffix = (String) json.get(PATH_SUFFIX_JSON);
784        Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix);
785        FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON));
786        long len = (Long) json.get(LENGTH_JSON);
787        String owner = (String) json.get(OWNER_JSON);
788        String group = (String) json.get(GROUP_JSON);
789        FsPermission permission =
790          new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8));
791        long aTime = (Long) json.get(ACCESS_TIME_JSON);
792        long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
793        long blockSize = (Long) json.get(BLOCK_SIZE_JSON);
794        short replication = ((Long) json.get(REPLICATION_JSON)).shortValue();
795        FileStatus fileStatus = null;
796    
797        switch (type) {
798          case FILE:
799          case DIRECTORY:
800            fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY),
801                                        replication, blockSize, mTime, aTime,
802                                        permission, owner, group, path);
803            break;
804          case SYMLINK:
805            Path symLink = null;
806            fileStatus = new FileStatus(len, false,
807                                        replication, blockSize, mTime, aTime,
808                                        permission, owner, group, symLink,
809                                        path);
810        }
811        return fileStatus;
812      }
813    
814      @Override
815      public ContentSummary getContentSummary(Path f) throws IOException {
816        Map<String, String> params = new HashMap<String, String>();
817        params.put(OP_PARAM, GetOpValues.GETCONTENTSUMMARY.toString());
818        HttpURLConnection conn = getConnection(HTTP_GET, params, f, true);
819        validateResponse(conn, HttpURLConnection.HTTP_OK);
820        JSONObject json = (JSONObject) ((JSONObject) jsonParse(conn)).get(CONTENT_SUMMARY_JSON);
821        return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON),
822                                  (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON),
823                                  (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON),
824                                  (Long) json.get(CONTENT_SUMMARY_QUOTA_JSON),
825                                  (Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON),
826                                  (Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON)
827        );
828      }
829    
830      @Override
831      public FileChecksum getFileChecksum(Path f) throws IOException {
832        Map<String, String> params = new HashMap<String, String>();
833        params.put(OP_PARAM, GetOpValues.GETFILECHECKSUM.toString());
834        HttpURLConnection conn = getConnection(HTTP_GET, params, f, true);
835        validateResponse(conn, HttpURLConnection.HTTP_OK);
836        final JSONObject json = (JSONObject) ((JSONObject) jsonParse(conn)).get(FILE_CHECKSUM_JSON);
837        return new FileChecksum() {
838          @Override
839          public String getAlgorithmName() {
840            return (String) json.get(CHECKSUM_ALGORITHM_JSON);
841          }
842    
843          @Override
844          public int getLength() {
845            return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue();
846          }
847    
848          @Override
849          public byte[] getBytes() {
850            return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON));
851          }
852    
853          @Override
854          public void write(DataOutput out) throws IOException {
855            throw new UnsupportedOperationException();
856          }
857    
858          @Override
859          public void readFields(DataInput in) throws IOException {
860            throw new UnsupportedOperationException();
861          }
862        };
863      }
864    
865    }