001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs.http.client;
019    
020    import org.apache.hadoop.classification.InterfaceAudience;
021    import org.apache.hadoop.conf.Configuration;
022    import org.apache.hadoop.fs.ContentSummary;
023    import org.apache.hadoop.fs.DelegationTokenRenewer;
024    import org.apache.hadoop.fs.FSDataInputStream;
025    import org.apache.hadoop.fs.FSDataOutputStream;
026    import org.apache.hadoop.fs.FileChecksum;
027    import org.apache.hadoop.fs.FileStatus;
028    import org.apache.hadoop.fs.FileSystem;
029    import org.apache.hadoop.fs.Path;
030    import org.apache.hadoop.fs.PositionedReadable;
031    import org.apache.hadoop.fs.Seekable;
032    import org.apache.hadoop.fs.permission.FsPermission;
033    import org.apache.hadoop.hdfs.DFSConfigKeys;
034    import org.apache.hadoop.net.NetUtils;
035    import org.apache.hadoop.security.UserGroupInformation;
036    import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
037    import org.apache.hadoop.security.authentication.client.Authenticator;
038    import org.apache.hadoop.security.token.Token;
039    import org.apache.hadoop.security.token.TokenIdentifier;
040    import org.apache.hadoop.util.Progressable;
041    import org.apache.hadoop.util.ReflectionUtils;
042    import org.apache.hadoop.util.StringUtils;
043    import org.json.simple.JSONArray;
044    import org.json.simple.JSONObject;
045    
046    import java.io.BufferedInputStream;
047    import java.io.BufferedOutputStream;
048    import java.io.DataInput;
049    import java.io.DataOutput;
050    import java.io.FileNotFoundException;
051    import java.io.FilterInputStream;
052    import java.io.IOException;
053    import java.io.InputStream;
054    import java.io.OutputStream;
055    import java.net.HttpURLConnection;
056    import java.net.InetSocketAddress;
057    import java.net.URI;
058    import java.net.URISyntaxException;
059    import java.net.URL;
060    import java.security.PrivilegedExceptionAction;
061    import java.text.MessageFormat;
062    import java.util.HashMap;
063    import java.util.Map;
064    import java.util.concurrent.Callable;
065    
066    /**
067     * HttpFSServer implementation of the FileSystemAccess FileSystem.
068     * <p/>
069     * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server.
070     */
071    @InterfaceAudience.Private
072    public class HttpFSFileSystem extends FileSystem
073      implements DelegationTokenRenewer.Renewable {
074    
075      public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME;
076    
077      public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION;
078    
079      public static final String SCHEME = "webhdfs";
080    
081      public static final String OP_PARAM = "op";
082      public static final String DO_AS_PARAM = "doas";
083      public static final String OVERWRITE_PARAM = "overwrite";
084      public static final String REPLICATION_PARAM = "replication";
085      public static final String BLOCKSIZE_PARAM = "blocksize";
086      public static final String PERMISSION_PARAM = "permission";
087      public static final String DESTINATION_PARAM = "destination";
088      public static final String RECURSIVE_PARAM = "recursive";
089      public static final String OWNER_PARAM = "owner";
090      public static final String GROUP_PARAM = "group";
091      public static final String MODIFICATION_TIME_PARAM = "modificationtime";
092      public static final String ACCESS_TIME_PARAM = "accesstime";
093    
094      public static final Short DEFAULT_PERMISSION = 0755;
095    
096      public static final String RENAME_JSON = "boolean";
097    
098      public static final String DELETE_JSON = "boolean";
099    
100      public static final String MKDIRS_JSON = "boolean";
101    
102      public static final String HOME_DIR_JSON = "Path";
103    
104      public static final String SET_REPLICATION_JSON = "boolean";
105    
106      public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream";
107    
108      public static enum FILE_TYPE {
109        FILE, DIRECTORY, SYMLINK;
110    
111        public static FILE_TYPE getType(FileStatus fileStatus) {
112          if (fileStatus.isFile()) {
113            return FILE;
114          }
115          if (fileStatus.isDirectory()) {
116            return DIRECTORY;
117          }
118          if (fileStatus.isSymlink()) {
119            return SYMLINK;
120          }
121          throw new IllegalArgumentException("Could not determine filetype for: " +
122                                             fileStatus.getPath());
123        }
124      }
125    
126      public static final String FILE_STATUSES_JSON = "FileStatuses";
127      public static final String FILE_STATUS_JSON = "FileStatus";
128      public static final String PATH_SUFFIX_JSON = "pathSuffix";
129      public static final String TYPE_JSON = "type";
130      public static final String LENGTH_JSON = "length";
131      public static final String OWNER_JSON = "owner";
132      public static final String GROUP_JSON = "group";
133      public static final String PERMISSION_JSON = "permission";
134      public static final String ACCESS_TIME_JSON = "accessTime";
135      public static final String MODIFICATION_TIME_JSON = "modificationTime";
136      public static final String BLOCK_SIZE_JSON = "blockSize";
137      public static final String REPLICATION_JSON = "replication";
138    
139      public static final String FILE_CHECKSUM_JSON = "FileChecksum";
140      public static final String CHECKSUM_ALGORITHM_JSON = "algorithm";
141      public static final String CHECKSUM_BYTES_JSON = "bytes";
142      public static final String CHECKSUM_LENGTH_JSON = "length";
143    
144      public static final String CONTENT_SUMMARY_JSON = "ContentSummary";
145      public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount";
146      public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount";
147      public static final String CONTENT_SUMMARY_LENGTH_JSON = "length";
148      public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota";
149      public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed";
150      public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota";
151    
152      public static final String ERROR_JSON = "RemoteException";
153      public static final String ERROR_EXCEPTION_JSON = "exception";
154      public static final String ERROR_CLASSNAME_JSON = "javaClassName";
155      public static final String ERROR_MESSAGE_JSON = "message";
156    
157      public static final int HTTP_TEMPORARY_REDIRECT = 307;
158    
159      private static final String HTTP_GET = "GET";
160      private static final String HTTP_PUT = "PUT";
161      private static final String HTTP_POST = "POST";
162      private static final String HTTP_DELETE = "DELETE";
163    
164      @InterfaceAudience.Private
165      public static enum Operation {
166        OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET),
167        GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET),
168        GETFILECHECKSUM(HTTP_GET),  GETFILEBLOCKLOCATIONS(HTTP_GET),
169        INSTRUMENTATION(HTTP_GET),
170        APPEND(HTTP_POST),
171        CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT),
172        SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT),
173        DELETE(HTTP_DELETE);
174    
175        private String httpMethod;
176    
177        Operation(String httpMethod) {
178          this.httpMethod = httpMethod;
179        }
180    
181        public String getMethod() {
182          return httpMethod;
183        }
184    
185      }
186    
187    
188      private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
189      private URI uri;
190      private InetSocketAddress httpFSAddr;
191      private Path workingDir;
192      private UserGroupInformation realUser;
193      private String doAs;
194      private Token<?> delegationToken;
195    
196      //This method enables handling UGI doAs with SPNEGO, we have to
197      //fallback to the realuser who logged in with Kerberos credentials
198      private <T> T doAsRealUserIfNecessary(final Callable<T> callable)
199        throws IOException {
200        try {
201          if (realUser.getShortUserName().equals(doAs)) {
202            return callable.call();
203          } else {
204            return realUser.doAs(new PrivilegedExceptionAction<T>() {
205              @Override
206              public T run() throws Exception {
207                return callable.call();
208              }
209            });
210          }
211        } catch (Exception ex) {
212          throw new IOException(ex.toString(), ex);
213        }
214      }
215    
216      /**
217       * Convenience method that creates a <code>HttpURLConnection</code> for the
218       * HttpFSServer file system operations.
219       * <p/>
220       * This methods performs and injects any needed authentication credentials
221       * via the {@link #getConnection(URL, String)} method
222       *
223       * @param method the HTTP method.
224       * @param params the query string parameters.
225       * @param path the file path
226       * @param makeQualified if the path should be 'makeQualified'
227       *
228       * @return a <code>HttpURLConnection</code> for the HttpFSServer server,
229       *         authenticated and ready to use for the specified path and file system operation.
230       *
231       * @throws IOException thrown if an IO error occurrs.
232       */
233      private HttpURLConnection getConnection(final String method,
234          Map<String, String> params, Path path, boolean makeQualified)
235          throws IOException {
236        if (!realUser.getShortUserName().equals(doAs)) {
237          params.put(DO_AS_PARAM, doAs);
238        }
239        HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken);
240        if (makeQualified) {
241          path = makeQualified(path);
242        }
243        final URL url = HttpFSUtils.createHttpURL(path, params);
244        return doAsRealUserIfNecessary(new Callable<HttpURLConnection>() {
245          @Override
246          public HttpURLConnection call() throws Exception {
247            return getConnection(url, method);
248          }
249        });
250      }
251    
252      /**
253       * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL.
254       * <p/>
255       * This methods performs and injects any needed authentication credentials.
256       *
257       * @param url url to connect to.
258       * @param method the HTTP method.
259       *
260       * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for
261       *         the specified path and file system operation.
262       *
263       * @throws IOException thrown if an IO error occurrs.
264       */
265      private HttpURLConnection getConnection(URL url, String method) throws IOException {
266        Class<? extends Authenticator> klass =
267          getConf().getClass("httpfs.authenticator.class",
268                             HttpFSKerberosAuthenticator.class, Authenticator.class);
269        Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf());
270        try {
271          HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken);
272          conn.setRequestMethod(method);
273          if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) {
274            conn.setDoOutput(true);
275          }
276          return conn;
277        } catch (Exception ex) {
278          throw new IOException(ex);
279        }
280      }
281    
282      /**
283       * Called after a new FileSystem instance is constructed.
284       *
285       * @param name a uri whose authority section names the host, port, etc. for this FileSystem
286       * @param conf the configuration
287       */
288      @Override
289      public void initialize(URI name, Configuration conf) throws IOException {
290        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
291    
292        //the real use is the one that has the Kerberos credentials needed for
293        //SPNEGO to work
294        realUser = ugi.getRealUser();
295        if (realUser == null) {
296          realUser = UserGroupInformation.getLoginUser();
297        }
298        doAs = ugi.getShortUserName();
299        super.initialize(name, conf);
300        try {
301          uri = new URI(name.getScheme() + "://" + name.getAuthority());
302          httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString());
303        } catch (URISyntaxException ex) {
304          throw new IOException(ex);
305        }
306      }
307    
308      @Override
309      public String getScheme() {
310        return SCHEME;
311      }
312    
313      /**
314       * Returns a URI whose scheme and authority identify this FileSystem.
315       *
316       * @return the URI whose scheme and authority identify this FileSystem.
317       */
318      @Override
319      public URI getUri() {
320        return uri;
321      }
322    
323      /**
324       * Get the default port for this file system.
325       * @return the default port or 0 if there isn't one
326       */
327      @Override
328      protected int getDefaultPort() {
329        return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
330            DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
331      }
332    
333      /**
334       * HttpFSServer subclass of the <code>FSDataInputStream</code>.
335       * <p/>
336       * This implementation does not support the
337       * <code>PositionReadable</code> and <code>Seekable</code> methods.
338       */
339      private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable {
340    
341        protected HttpFSDataInputStream(InputStream in, int bufferSize) {
342          super(new BufferedInputStream(in, bufferSize));
343        }
344    
345        @Override
346        public int read(long position, byte[] buffer, int offset, int length) throws IOException {
347          throw new UnsupportedOperationException();
348        }
349    
350        @Override
351        public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
352          throw new UnsupportedOperationException();
353        }
354    
355        @Override
356        public void readFully(long position, byte[] buffer) throws IOException {
357          throw new UnsupportedOperationException();
358        }
359    
360        @Override
361        public void seek(long pos) throws IOException {
362          throw new UnsupportedOperationException();
363        }
364    
365        @Override
366        public long getPos() throws IOException {
367          throw new UnsupportedOperationException();
368        }
369    
370        @Override
371        public boolean seekToNewSource(long targetPos) throws IOException {
372          throw new UnsupportedOperationException();
373        }
374      }
375    
376      /**
377       * Opens an FSDataInputStream at the indicated Path.
378       * </p>
379       * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the
380       * <code>PositionReadable</code> and <code>Seekable</code> methods.
381       *
382       * @param f the file name to open
383       * @param bufferSize the size of the buffer to be used.
384       */
385      @Override
386      public FSDataInputStream open(Path f, int bufferSize) throws IOException {
387        Map<String, String> params = new HashMap<String, String>();
388        params.put(OP_PARAM, Operation.OPEN.toString());
389        HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params,
390                                               f, true);
391        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
392        return new FSDataInputStream(
393          new HttpFSDataInputStream(conn.getInputStream(), bufferSize));
394      }
395    
396      /**
397       * HttpFSServer subclass of the <code>FSDataOutputStream</code>.
398       * <p/>
399       * This implementation closes the underlying HTTP connection validating the Http connection status
400       * at closing time.
401       */
402      private static class HttpFSDataOutputStream extends FSDataOutputStream {
403        private HttpURLConnection conn;
404        private int closeStatus;
405    
406        public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats)
407          throws IOException {
408          super(out, stats);
409          this.conn = conn;
410          this.closeStatus = closeStatus;
411        }
412    
413        @Override
414        public void close() throws IOException {
415          try {
416            super.close();
417          } finally {
418            HttpFSUtils.validateResponse(conn, closeStatus);
419          }
420        }
421    
422      }
423    
424      /**
425       * Converts a <code>FsPermission</code> to a Unix octal representation.
426       *
427       * @param p the permission.
428       *
429       * @return the Unix string symbolic reprentation.
430       */
431      public static String permissionToString(FsPermission p) {
432        return  Integer.toString((p == null) ? DEFAULT_PERMISSION : p.toShort(), 8);
433      }
434    
435      /*
436       * Common handling for uploading data for create and append operations.
437       */
438      private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params,
439                                            int bufferSize, int expectedStatus) throws IOException {
440        HttpURLConnection conn = getConnection(method, params, f, true);
441        conn.setInstanceFollowRedirects(false);
442        boolean exceptionAlreadyHandled = false;
443        try {
444          if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) {
445            exceptionAlreadyHandled = true;
446            String location = conn.getHeaderField("Location");
447            if (location != null) {
448              conn = getConnection(new URL(location), method);
449              conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE);
450              try {
451                OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize);
452                return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics);
453              } catch (IOException ex) {
454                HttpFSUtils.validateResponse(conn, expectedStatus);
455                throw ex;
456              }
457            } else {
458              HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
459              throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]");
460            }
461          } else {
462            throw new IOException(
463              MessageFormat.format("Expected HTTP status was [307], received [{0}]",
464                                   conn.getResponseCode()));
465          }
466        } catch (IOException ex) {
467          if (exceptionAlreadyHandled) {
468            throw ex;
469          } else {
470            HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
471            throw ex;
472          }
473        }
474      }
475    
476    
477      /**
478       * Opens an FSDataOutputStream at the indicated Path with write-progress
479       * reporting.
480       * <p/>
481       * IMPORTANT: The <code>Progressable</code> parameter is not used.
482       *
483       * @param f the file name to open.
484       * @param permission file permission.
485       * @param overwrite if a file with this name already exists, then if true,
486       * the file will be overwritten, and if false an error will be thrown.
487       * @param bufferSize the size of the buffer to be used.
488       * @param replication required block replication for the file.
489       * @param blockSize block size.
490       * @param progress progressable.
491       *
492       * @throws IOException
493       * @see #setPermission(Path, FsPermission)
494       */
495      @Override
496      public FSDataOutputStream create(Path f, FsPermission permission,
497                                       boolean overwrite, int bufferSize,
498                                       short replication, long blockSize,
499                                       Progressable progress) throws IOException {
500        Map<String, String> params = new HashMap<String, String>();
501        params.put(OP_PARAM, Operation.CREATE.toString());
502        params.put(OVERWRITE_PARAM, Boolean.toString(overwrite));
503        params.put(REPLICATION_PARAM, Short.toString(replication));
504        params.put(BLOCKSIZE_PARAM, Long.toString(blockSize));
505        params.put(PERMISSION_PARAM, permissionToString(permission));
506        return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize,
507                          HttpURLConnection.HTTP_CREATED);
508      }
509    
510    
511      /**
512       * Append to an existing file (optional operation).
513       * <p/>
514       * IMPORTANT: The <code>Progressable</code> parameter is not used.
515       *
516       * @param f the existing file to be appended.
517       * @param bufferSize the size of the buffer to be used.
518       * @param progress for reporting progress if it is not null.
519       *
520       * @throws IOException
521       */
522      @Override
523      public FSDataOutputStream append(Path f, int bufferSize,
524                                       Progressable progress) throws IOException {
525        Map<String, String> params = new HashMap<String, String>();
526        params.put(OP_PARAM, Operation.APPEND.toString());
527        return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize,
528                          HttpURLConnection.HTTP_OK);
529      }
530    
531      /**
532       * Renames Path src to Path dst.  Can take place on local fs
533       * or remote DFS.
534       */
535      @Override
536      public boolean rename(Path src, Path dst) throws IOException {
537        Map<String, String> params = new HashMap<String, String>();
538        params.put(OP_PARAM, Operation.RENAME.toString());
539        params.put(DESTINATION_PARAM, dst.toString());
540        HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(),
541                                               params, src, true);
542        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
543        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
544        return (Boolean) json.get(RENAME_JSON);
545      }
546    
547      /**
548       * Delete a file.
549       *
550       * @deprecated Use delete(Path, boolean) instead
551       */
552      @SuppressWarnings({"deprecation"})
553      @Deprecated
554      @Override
555      public boolean delete(Path f) throws IOException {
556        return delete(f, false);
557      }
558    
559      /**
560       * Delete a file.
561       *
562       * @param f the path to delete.
563       * @param recursive if path is a directory and set to
564       * true, the directory is deleted else throws an exception. In
565       * case of a file the recursive can be set to either true or false.
566       *
567       * @return true if delete is successful else false.
568       *
569       * @throws IOException
570       */
571      @Override
572      public boolean delete(Path f, boolean recursive) throws IOException {
573        Map<String, String> params = new HashMap<String, String>();
574        params.put(OP_PARAM, Operation.DELETE.toString());
575        params.put(RECURSIVE_PARAM, Boolean.toString(recursive));
576        HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(),
577                                               params, f, true);
578        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
579        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
580        return (Boolean) json.get(DELETE_JSON);
581      }
582    
583      /**
584       * List the statuses of the files/directories in the given path if the path is
585       * a directory.
586       *
587       * @param f given path
588       *
589       * @return the statuses of the files/directories in the given patch
590       *
591       * @throws IOException
592       */
593      @Override
594      public FileStatus[] listStatus(Path f) throws IOException {
595        Map<String, String> params = new HashMap<String, String>();
596        params.put(OP_PARAM, Operation.LISTSTATUS.toString());
597        HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(),
598                                               params, f, true);
599        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
600        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
601        json = (JSONObject) json.get(FILE_STATUSES_JSON);
602        JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON);
603        FileStatus[] array = new FileStatus[jsonArray.size()];
604        f = makeQualified(f);
605        for (int i = 0; i < jsonArray.size(); i++) {
606          array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i));
607        }
608        return array;
609      }
610    
611      /**
612       * Set the current working directory for the given file system. All relative
613       * paths will be resolved relative to it.
614       *
615       * @param newDir new directory.
616       */
617      @Override
618      public void setWorkingDirectory(Path newDir) {
619        workingDir = newDir;
620      }
621    
622      /**
623       * Get the current working directory for the given file system
624       *
625       * @return the directory pathname
626       */
627      @Override
628      public Path getWorkingDirectory() {
629        if (workingDir == null) {
630          workingDir = getHomeDirectory();
631        }
632        return workingDir;
633      }
634    
635      /**
636       * Make the given file and all non-existent parents into
637       * directories. Has the semantics of Unix 'mkdir -p'.
638       * Existence of the directory hierarchy is not an error.
639       */
640      @Override
641      public boolean mkdirs(Path f, FsPermission permission) throws IOException {
642        Map<String, String> params = new HashMap<String, String>();
643        params.put(OP_PARAM, Operation.MKDIRS.toString());
644        params.put(PERMISSION_PARAM, permissionToString(permission));
645        HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(),
646                                               params, f, true);
647        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
648        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
649        return (Boolean) json.get(MKDIRS_JSON);
650      }
651    
652      /**
653       * Return a file status object that represents the path.
654       *
655       * @param f The path we want information from
656       *
657       * @return a FileStatus object
658       *
659       * @throws FileNotFoundException when the path does not exist;
660       * IOException see specific implementation
661       */
662      @Override
663      public FileStatus getFileStatus(Path f) throws IOException {
664        Map<String, String> params = new HashMap<String, String>();
665        params.put(OP_PARAM, Operation.GETFILESTATUS.toString());
666        HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(),
667                                               params, f, true);
668        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
669        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
670        json = (JSONObject) json.get(FILE_STATUS_JSON);
671        f = makeQualified(f);
672        return createFileStatus(f, json);
673      }
674    
675      /**
676       * Return the current user's home directory in this filesystem.
677       * The default implementation returns "/user/$USER/".
678       */
679      @Override
680      public Path getHomeDirectory() {
681        Map<String, String> params = new HashMap<String, String>();
682        params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString());
683        try {
684          HttpURLConnection conn =
685            getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params,
686                          new Path(getUri().toString(), "/"), false);
687          HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
688          JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
689          return new Path((String) json.get(HOME_DIR_JSON));
690        } catch (IOException ex) {
691          throw new RuntimeException(ex);
692        }
693      }
694    
695      /**
696       * Set owner of a path (i.e. a file or a directory).
697       * The parameters username and groupname cannot both be null.
698       *
699       * @param p The path
700       * @param username If it is null, the original username remains unchanged.
701       * @param groupname If it is null, the original groupname remains unchanged.
702       */
703      @Override
704      public void setOwner(Path p, String username, String groupname)
705        throws IOException {
706        Map<String, String> params = new HashMap<String, String>();
707        params.put(OP_PARAM, Operation.SETOWNER.toString());
708        params.put(OWNER_PARAM, username);
709        params.put(GROUP_PARAM, groupname);
710        HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(),
711                                               params, p, true);
712        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
713      }
714    
715      /**
716       * Set permission of a path.
717       *
718       * @param p path.
719       * @param permission permission.
720       */
721      @Override
722      public void setPermission(Path p, FsPermission permission) throws IOException {
723        Map<String, String> params = new HashMap<String, String>();
724        params.put(OP_PARAM, Operation.SETPERMISSION.toString());
725        params.put(PERMISSION_PARAM, permissionToString(permission));
726        HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true);
727        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
728      }
729    
730      /**
731       * Set access time of a file
732       *
733       * @param p The path
734       * @param mtime Set the modification time of this file.
735       * The number of milliseconds since Jan 1, 1970.
736       * A value of -1 means that this call should not set modification time.
737       * @param atime Set the access time of this file.
738       * The number of milliseconds since Jan 1, 1970.
739       * A value of -1 means that this call should not set access time.
740       */
741      @Override
742      public void setTimes(Path p, long mtime, long atime) throws IOException {
743        Map<String, String> params = new HashMap<String, String>();
744        params.put(OP_PARAM, Operation.SETTIMES.toString());
745        params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime));
746        params.put(ACCESS_TIME_PARAM, Long.toString(atime));
747        HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(),
748                                               params, p, true);
749        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
750      }
751    
752      /**
753       * Set replication for an existing file.
754       *
755       * @param src file name
756       * @param replication new replication
757       *
758       * @return true if successful;
759       *         false if file does not exist or is a directory
760       *
761       * @throws IOException
762       */
763      @Override
764      public boolean setReplication(Path src, short replication)
765        throws IOException {
766        Map<String, String> params = new HashMap<String, String>();
767        params.put(OP_PARAM, Operation.SETREPLICATION.toString());
768        params.put(REPLICATION_PARAM, Short.toString(replication));
769        HttpURLConnection conn =
770          getConnection(Operation.SETREPLICATION.getMethod(), params, src, true);
771        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
772        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
773        return (Boolean) json.get(SET_REPLICATION_JSON);
774      }
775    
776      private FileStatus createFileStatus(Path parent, JSONObject json) {
777        String pathSuffix = (String) json.get(PATH_SUFFIX_JSON);
778        Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix);
779        FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON));
780        long len = (Long) json.get(LENGTH_JSON);
781        String owner = (String) json.get(OWNER_JSON);
782        String group = (String) json.get(GROUP_JSON);
783        FsPermission permission =
784          new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8));
785        long aTime = (Long) json.get(ACCESS_TIME_JSON);
786        long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
787        long blockSize = (Long) json.get(BLOCK_SIZE_JSON);
788        short replication = ((Long) json.get(REPLICATION_JSON)).shortValue();
789        FileStatus fileStatus = null;
790    
791        switch (type) {
792          case FILE:
793          case DIRECTORY:
794            fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY),
795                                        replication, blockSize, mTime, aTime,
796                                        permission, owner, group, path);
797            break;
798          case SYMLINK:
799            Path symLink = null;
800            fileStatus = new FileStatus(len, false,
801                                        replication, blockSize, mTime, aTime,
802                                        permission, owner, group, symLink,
803                                        path);
804        }
805        return fileStatus;
806      }
807    
808      @Override
809      public ContentSummary getContentSummary(Path f) throws IOException {
810        Map<String, String> params = new HashMap<String, String>();
811        params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString());
812        HttpURLConnection conn =
813          getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true);
814        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
815        JSONObject json = (JSONObject) ((JSONObject)
816          HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON);
817        return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON),
818                                  (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON),
819                                  (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON),
820                                  (Long) json.get(CONTENT_SUMMARY_QUOTA_JSON),
821                                  (Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON),
822                                  (Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON)
823        );
824      }
825    
826      @Override
827      public FileChecksum getFileChecksum(Path f) throws IOException {
828        Map<String, String> params = new HashMap<String, String>();
829        params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString());
830        HttpURLConnection conn =
831          getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true);
832        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
833        final JSONObject json = (JSONObject) ((JSONObject)
834          HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON);
835        return new FileChecksum() {
836          @Override
837          public String getAlgorithmName() {
838            return (String) json.get(CHECKSUM_ALGORITHM_JSON);
839          }
840    
841          @Override
842          public int getLength() {
843            return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue();
844          }
845    
846          @Override
847          public byte[] getBytes() {
848            return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON));
849          }
850    
851          @Override
852          public void write(DataOutput out) throws IOException {
853            throw new UnsupportedOperationException();
854          }
855    
856          @Override
857          public void readFields(DataInput in) throws IOException {
858            throw new UnsupportedOperationException();
859          }
860        };
861      }
862    
863    
864      @Override
865      public Token<?> getDelegationToken(final String renewer)
866        throws IOException {
867        return doAsRealUserIfNecessary(new Callable<Token<?>>() {
868          @Override
869          public Token<?> call() throws Exception {
870            return HttpFSKerberosAuthenticator.
871              getDelegationToken(uri, httpFSAddr, authToken, renewer);
872          }
873        });
874      }
875    
876      public long renewDelegationToken(final Token<?> token) throws IOException {
877        return doAsRealUserIfNecessary(new Callable<Long>() {
878          @Override
879          public Long call() throws Exception {
880            return HttpFSKerberosAuthenticator.
881              renewDelegationToken(uri,  authToken, token);
882          }
883        });
884      }
885    
886      public void cancelDelegationToken(final Token<?> token) throws IOException {
887        HttpFSKerberosAuthenticator.
888          cancelDelegationToken(uri, authToken, token);
889      }
890    
891      @Override
892      public Token<?> getRenewToken() {
893        return delegationToken;
894      }
895    
896      @Override
897      public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
898        delegationToken = token;
899      }
900    
901    }