001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs.http.client;
019    
020    import java.util.ArrayList;
021    import java.util.List;
022    import org.apache.hadoop.classification.InterfaceAudience;
023    import org.apache.hadoop.conf.Configuration;
024    import org.apache.hadoop.fs.ContentSummary;
025    import org.apache.hadoop.fs.DelegationTokenRenewer;
026    import org.apache.hadoop.fs.FSDataInputStream;
027    import org.apache.hadoop.fs.FSDataOutputStream;
028    import org.apache.hadoop.fs.FileChecksum;
029    import org.apache.hadoop.fs.FileStatus;
030    import org.apache.hadoop.fs.FileSystem;
031    import org.apache.hadoop.fs.Path;
032    import org.apache.hadoop.fs.PositionedReadable;
033    import org.apache.hadoop.fs.Seekable;
034    import org.apache.hadoop.fs.permission.FsPermission;
035    import org.apache.hadoop.hdfs.DFSConfigKeys;
036    import org.apache.hadoop.net.NetUtils;
037    import org.apache.hadoop.security.UserGroupInformation;
038    import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
039    import org.apache.hadoop.security.authentication.client.Authenticator;
040    import org.apache.hadoop.security.token.Token;
041    import org.apache.hadoop.security.token.TokenIdentifier;
042    import org.apache.hadoop.util.Progressable;
043    import org.apache.hadoop.util.ReflectionUtils;
044    import org.apache.hadoop.util.StringUtils;
045    import org.json.simple.JSONArray;
046    import org.json.simple.JSONObject;
047    
048    import java.io.BufferedInputStream;
049    import java.io.BufferedOutputStream;
050    import java.io.DataInput;
051    import java.io.DataOutput;
052    import java.io.FileNotFoundException;
053    import java.io.FilterInputStream;
054    import java.io.IOException;
055    import java.io.InputStream;
056    import java.io.OutputStream;
057    import java.net.HttpURLConnection;
058    import java.net.InetSocketAddress;
059    import java.net.URI;
060    import java.net.URISyntaxException;
061    import java.net.URL;
062    import java.security.PrivilegedExceptionAction;
063    import java.text.MessageFormat;
064    import java.util.HashMap;
065    import java.util.Map;
066    import java.util.concurrent.Callable;
067    
068    /**
069     * HttpFSServer implementation of the FileSystemAccess FileSystem.
070     * <p/>
071     * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server.
072     */
073    @InterfaceAudience.Private
074    public class HttpFSFileSystem extends FileSystem
075      implements DelegationTokenRenewer.Renewable {
076    
077      public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME;
078    
079      public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION;
080    
081      public static final String SCHEME = "webhdfs";
082    
083      public static final String OP_PARAM = "op";
084      public static final String DO_AS_PARAM = "doas";
085      public static final String OVERWRITE_PARAM = "overwrite";
086      public static final String REPLICATION_PARAM = "replication";
087      public static final String BLOCKSIZE_PARAM = "blocksize";
088      public static final String PERMISSION_PARAM = "permission";
089      public static final String DESTINATION_PARAM = "destination";
090      public static final String RECURSIVE_PARAM = "recursive";
091      public static final String SOURCES_PARAM = "sources";
092      public static final String OWNER_PARAM = "owner";
093      public static final String GROUP_PARAM = "group";
094      public static final String MODIFICATION_TIME_PARAM = "modificationtime";
095      public static final String ACCESS_TIME_PARAM = "accesstime";
096    
097      public static final Short DEFAULT_PERMISSION = 0755;
098    
099      public static final String RENAME_JSON = "boolean";
100    
101      public static final String DELETE_JSON = "boolean";
102    
103      public static final String MKDIRS_JSON = "boolean";
104    
105      public static final String HOME_DIR_JSON = "Path";
106    
107      public static final String SET_REPLICATION_JSON = "boolean";
108    
109      public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream";
110    
111      public static enum FILE_TYPE {
112        FILE, DIRECTORY, SYMLINK;
113    
114        public static FILE_TYPE getType(FileStatus fileStatus) {
115          if (fileStatus.isFile()) {
116            return FILE;
117          }
118          if (fileStatus.isDirectory()) {
119            return DIRECTORY;
120          }
121          if (fileStatus.isSymlink()) {
122            return SYMLINK;
123          }
124          throw new IllegalArgumentException("Could not determine filetype for: " +
125                                             fileStatus.getPath());
126        }
127      }
128    
129      public static final String FILE_STATUSES_JSON = "FileStatuses";
130      public static final String FILE_STATUS_JSON = "FileStatus";
131      public static final String PATH_SUFFIX_JSON = "pathSuffix";
132      public static final String TYPE_JSON = "type";
133      public static final String LENGTH_JSON = "length";
134      public static final String OWNER_JSON = "owner";
135      public static final String GROUP_JSON = "group";
136      public static final String PERMISSION_JSON = "permission";
137      public static final String ACCESS_TIME_JSON = "accessTime";
138      public static final String MODIFICATION_TIME_JSON = "modificationTime";
139      public static final String BLOCK_SIZE_JSON = "blockSize";
140      public static final String REPLICATION_JSON = "replication";
141    
142      public static final String FILE_CHECKSUM_JSON = "FileChecksum";
143      public static final String CHECKSUM_ALGORITHM_JSON = "algorithm";
144      public static final String CHECKSUM_BYTES_JSON = "bytes";
145      public static final String CHECKSUM_LENGTH_JSON = "length";
146    
147      public static final String CONTENT_SUMMARY_JSON = "ContentSummary";
148      public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount";
149      public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount";
150      public static final String CONTENT_SUMMARY_LENGTH_JSON = "length";
151      public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota";
152      public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed";
153      public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota";
154    
155      public static final String ERROR_JSON = "RemoteException";
156      public static final String ERROR_EXCEPTION_JSON = "exception";
157      public static final String ERROR_CLASSNAME_JSON = "javaClassName";
158      public static final String ERROR_MESSAGE_JSON = "message";
159    
160      public static final int HTTP_TEMPORARY_REDIRECT = 307;
161    
162      private static final String HTTP_GET = "GET";
163      private static final String HTTP_PUT = "PUT";
164      private static final String HTTP_POST = "POST";
165      private static final String HTTP_DELETE = "DELETE";
166    
167      @InterfaceAudience.Private
168      public static enum Operation {
169        OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET),
170        GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET),
171        GETFILECHECKSUM(HTTP_GET),  GETFILEBLOCKLOCATIONS(HTTP_GET),
172        INSTRUMENTATION(HTTP_GET),
173        APPEND(HTTP_POST), CONCAT(HTTP_POST),
174        CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT),
175        SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT),
176        DELETE(HTTP_DELETE);
177    
178        private String httpMethod;
179    
180        Operation(String httpMethod) {
181          this.httpMethod = httpMethod;
182        }
183    
184        public String getMethod() {
185          return httpMethod;
186        }
187    
188      }
189    
190    
191      private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
192      private URI uri;
193      private InetSocketAddress httpFSAddr;
194      private Path workingDir;
195      private UserGroupInformation realUser;
196      private String doAs;
197      private Token<?> delegationToken;
198    
199      //This method enables handling UGI doAs with SPNEGO, we have to
200      //fallback to the realuser who logged in with Kerberos credentials
201      private <T> T doAsRealUserIfNecessary(final Callable<T> callable)
202        throws IOException {
203        try {
204          if (realUser.getShortUserName().equals(doAs)) {
205            return callable.call();
206          } else {
207            return realUser.doAs(new PrivilegedExceptionAction<T>() {
208              @Override
209              public T run() throws Exception {
210                return callable.call();
211              }
212            });
213          }
214        } catch (Exception ex) {
215          throw new IOException(ex.toString(), ex);
216        }
217      }
218    
219      /**
220       * Convenience method that creates a <code>HttpURLConnection</code> for the
221       * HttpFSServer file system operations.
222       * <p/>
223       * This methods performs and injects any needed authentication credentials
224       * via the {@link #getConnection(URL, String)} method
225       *
226       * @param method the HTTP method.
227       * @param params the query string parameters.
228       * @param path the file path
229       * @param makeQualified if the path should be 'makeQualified'
230       *
231       * @return a <code>HttpURLConnection</code> for the HttpFSServer server,
232       *         authenticated and ready to use for the specified path and file system operation.
233       *
234       * @throws IOException thrown if an IO error occurrs.
235       */
236      private HttpURLConnection getConnection(final String method,
237          Map<String, String> params, Path path, boolean makeQualified)
238          throws IOException {
239        if (!realUser.getShortUserName().equals(doAs)) {
240          params.put(DO_AS_PARAM, doAs);
241        }
242        HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken);
243        if (makeQualified) {
244          path = makeQualified(path);
245        }
246        final URL url = HttpFSUtils.createHttpURL(path, params);
247        return doAsRealUserIfNecessary(new Callable<HttpURLConnection>() {
248          @Override
249          public HttpURLConnection call() throws Exception {
250            return getConnection(url, method);
251          }
252        });
253      }
254    
255      /**
256       * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL.
257       * <p/>
258       * This methods performs and injects any needed authentication credentials.
259       *
260       * @param url url to connect to.
261       * @param method the HTTP method.
262       *
263       * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for
264       *         the specified path and file system operation.
265       *
266       * @throws IOException thrown if an IO error occurrs.
267       */
268      private HttpURLConnection getConnection(URL url, String method) throws IOException {
269        Class<? extends Authenticator> klass =
270          getConf().getClass("httpfs.authenticator.class",
271                             HttpFSKerberosAuthenticator.class, Authenticator.class);
272        Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf());
273        try {
274          HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken);
275          conn.setRequestMethod(method);
276          if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) {
277            conn.setDoOutput(true);
278          }
279          return conn;
280        } catch (Exception ex) {
281          throw new IOException(ex);
282        }
283      }
284    
285      /**
286       * Called after a new FileSystem instance is constructed.
287       *
288       * @param name a uri whose authority section names the host, port, etc. for this FileSystem
289       * @param conf the configuration
290       */
291      @Override
292      public void initialize(URI name, Configuration conf) throws IOException {
293        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
294    
295        //the real use is the one that has the Kerberos credentials needed for
296        //SPNEGO to work
297        realUser = ugi.getRealUser();
298        if (realUser == null) {
299          realUser = UserGroupInformation.getLoginUser();
300        }
301        doAs = ugi.getShortUserName();
302        super.initialize(name, conf);
303        try {
304          uri = new URI(name.getScheme() + "://" + name.getAuthority());
305          httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString());
306        } catch (URISyntaxException ex) {
307          throw new IOException(ex);
308        }
309      }
310    
311      @Override
312      public String getScheme() {
313        return SCHEME;
314      }
315    
316      /**
317       * Returns a URI whose scheme and authority identify this FileSystem.
318       *
319       * @return the URI whose scheme and authority identify this FileSystem.
320       */
321      @Override
322      public URI getUri() {
323        return uri;
324      }
325    
326      /**
327       * Get the default port for this file system.
328       * @return the default port or 0 if there isn't one
329       */
330      @Override
331      protected int getDefaultPort() {
332        return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
333            DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
334      }
335    
336      /**
337       * HttpFSServer subclass of the <code>FSDataInputStream</code>.
338       * <p/>
339       * This implementation does not support the
340       * <code>PositionReadable</code> and <code>Seekable</code> methods.
341       */
342      private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable {
343    
344        protected HttpFSDataInputStream(InputStream in, int bufferSize) {
345          super(new BufferedInputStream(in, bufferSize));
346        }
347    
348        @Override
349        public int read(long position, byte[] buffer, int offset, int length) throws IOException {
350          throw new UnsupportedOperationException();
351        }
352    
353        @Override
354        public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
355          throw new UnsupportedOperationException();
356        }
357    
358        @Override
359        public void readFully(long position, byte[] buffer) throws IOException {
360          throw new UnsupportedOperationException();
361        }
362    
363        @Override
364        public void seek(long pos) throws IOException {
365          throw new UnsupportedOperationException();
366        }
367    
368        @Override
369        public long getPos() throws IOException {
370          throw new UnsupportedOperationException();
371        }
372    
373        @Override
374        public boolean seekToNewSource(long targetPos) throws IOException {
375          throw new UnsupportedOperationException();
376        }
377      }
378    
379      /**
380       * Opens an FSDataInputStream at the indicated Path.
381       * </p>
382       * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the
383       * <code>PositionReadable</code> and <code>Seekable</code> methods.
384       *
385       * @param f the file name to open
386       * @param bufferSize the size of the buffer to be used.
387       */
388      @Override
389      public FSDataInputStream open(Path f, int bufferSize) throws IOException {
390        Map<String, String> params = new HashMap<String, String>();
391        params.put(OP_PARAM, Operation.OPEN.toString());
392        HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params,
393                                               f, true);
394        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
395        return new FSDataInputStream(
396          new HttpFSDataInputStream(conn.getInputStream(), bufferSize));
397      }
398    
399      /**
400       * HttpFSServer subclass of the <code>FSDataOutputStream</code>.
401       * <p/>
402       * This implementation closes the underlying HTTP connection validating the Http connection status
403       * at closing time.
404       */
405      private static class HttpFSDataOutputStream extends FSDataOutputStream {
406        private HttpURLConnection conn;
407        private int closeStatus;
408    
409        public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats)
410          throws IOException {
411          super(out, stats);
412          this.conn = conn;
413          this.closeStatus = closeStatus;
414        }
415    
416        @Override
417        public void close() throws IOException {
418          try {
419            super.close();
420          } finally {
421            HttpFSUtils.validateResponse(conn, closeStatus);
422          }
423        }
424    
425      }
426    
427      /**
428       * Converts a <code>FsPermission</code> to a Unix octal representation.
429       *
430       * @param p the permission.
431       *
432       * @return the Unix string symbolic reprentation.
433       */
434      public static String permissionToString(FsPermission p) {
435        return  Integer.toString((p == null) ? DEFAULT_PERMISSION : p.toShort(), 8);
436      }
437    
438      /*
439       * Common handling for uploading data for create and append operations.
440       */
441      private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params,
442                                            int bufferSize, int expectedStatus) throws IOException {
443        HttpURLConnection conn = getConnection(method, params, f, true);
444        conn.setInstanceFollowRedirects(false);
445        boolean exceptionAlreadyHandled = false;
446        try {
447          if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) {
448            exceptionAlreadyHandled = true;
449            String location = conn.getHeaderField("Location");
450            if (location != null) {
451              conn = getConnection(new URL(location), method);
452              conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE);
453              try {
454                OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize);
455                return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics);
456              } catch (IOException ex) {
457                HttpFSUtils.validateResponse(conn, expectedStatus);
458                throw ex;
459              }
460            } else {
461              HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
462              throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]");
463            }
464          } else {
465            throw new IOException(
466              MessageFormat.format("Expected HTTP status was [307], received [{0}]",
467                                   conn.getResponseCode()));
468          }
469        } catch (IOException ex) {
470          if (exceptionAlreadyHandled) {
471            throw ex;
472          } else {
473            HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
474            throw ex;
475          }
476        }
477      }
478    
479    
480      /**
481       * Opens an FSDataOutputStream at the indicated Path with write-progress
482       * reporting.
483       * <p/>
484       * IMPORTANT: The <code>Progressable</code> parameter is not used.
485       *
486       * @param f the file name to open.
487       * @param permission file permission.
488       * @param overwrite if a file with this name already exists, then if true,
489       * the file will be overwritten, and if false an error will be thrown.
490       * @param bufferSize the size of the buffer to be used.
491       * @param replication required block replication for the file.
492       * @param blockSize block size.
493       * @param progress progressable.
494       *
495       * @throws IOException
496       * @see #setPermission(Path, FsPermission)
497       */
498      @Override
499      public FSDataOutputStream create(Path f, FsPermission permission,
500                                       boolean overwrite, int bufferSize,
501                                       short replication, long blockSize,
502                                       Progressable progress) throws IOException {
503        Map<String, String> params = new HashMap<String, String>();
504        params.put(OP_PARAM, Operation.CREATE.toString());
505        params.put(OVERWRITE_PARAM, Boolean.toString(overwrite));
506        params.put(REPLICATION_PARAM, Short.toString(replication));
507        params.put(BLOCKSIZE_PARAM, Long.toString(blockSize));
508        params.put(PERMISSION_PARAM, permissionToString(permission));
509        return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize,
510                          HttpURLConnection.HTTP_CREATED);
511      }
512    
513    
514      /**
515       * Append to an existing file (optional operation).
516       * <p/>
517       * IMPORTANT: The <code>Progressable</code> parameter is not used.
518       *
519       * @param f the existing file to be appended.
520       * @param bufferSize the size of the buffer to be used.
521       * @param progress for reporting progress if it is not null.
522       *
523       * @throws IOException
524       */
525      @Override
526      public FSDataOutputStream append(Path f, int bufferSize,
527                                       Progressable progress) throws IOException {
528        Map<String, String> params = new HashMap<String, String>();
529        params.put(OP_PARAM, Operation.APPEND.toString());
530        return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize,
531                          HttpURLConnection.HTTP_OK);
532      }
533    
534      /**
535       * Concat existing files together.
536       * @param f the path to the target destination.
537       * @param psrcs the paths to the sources to use for the concatenation.
538       *
539       * @throws IOException
540       */
541      @Override
542      public void concat(Path f, Path[] psrcs) throws IOException {
543        List<String> strPaths = new ArrayList<String>(psrcs.length);
544        for(Path psrc : psrcs) {
545          strPaths.add(psrc.toUri().getPath());
546        }
547        String srcs = StringUtils.join(",", strPaths);
548    
549        Map<String, String> params = new HashMap<String, String>();
550        params.put(OP_PARAM, Operation.CONCAT.toString());
551        params.put(SOURCES_PARAM, srcs);
552        HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(),
553            params, f, true);
554        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
555      }
556    
557      /**
558       * Renames Path src to Path dst.  Can take place on local fs
559       * or remote DFS.
560       */
561      @Override
562      public boolean rename(Path src, Path dst) throws IOException {
563        Map<String, String> params = new HashMap<String, String>();
564        params.put(OP_PARAM, Operation.RENAME.toString());
565        params.put(DESTINATION_PARAM, dst.toString());
566        HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(),
567                                               params, src, true);
568        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
569        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
570        return (Boolean) json.get(RENAME_JSON);
571      }
572    
573      /**
574       * Delete a file.
575       *
576       * @deprecated Use delete(Path, boolean) instead
577       */
578      @SuppressWarnings({"deprecation"})
579      @Deprecated
580      @Override
581      public boolean delete(Path f) throws IOException {
582        return delete(f, false);
583      }
584    
585      /**
586       * Delete a file.
587       *
588       * @param f the path to delete.
589       * @param recursive if path is a directory and set to
590       * true, the directory is deleted else throws an exception. In
591       * case of a file the recursive can be set to either true or false.
592       *
593       * @return true if delete is successful else false.
594       *
595       * @throws IOException
596       */
597      @Override
598      public boolean delete(Path f, boolean recursive) throws IOException {
599        Map<String, String> params = new HashMap<String, String>();
600        params.put(OP_PARAM, Operation.DELETE.toString());
601        params.put(RECURSIVE_PARAM, Boolean.toString(recursive));
602        HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(),
603                                               params, f, true);
604        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
605        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
606        return (Boolean) json.get(DELETE_JSON);
607      }
608    
609      /**
610       * List the statuses of the files/directories in the given path if the path is
611       * a directory.
612       *
613       * @param f given path
614       *
615       * @return the statuses of the files/directories in the given patch
616       *
617       * @throws IOException
618       */
619      @Override
620      public FileStatus[] listStatus(Path f) throws IOException {
621        Map<String, String> params = new HashMap<String, String>();
622        params.put(OP_PARAM, Operation.LISTSTATUS.toString());
623        HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(),
624                                               params, f, true);
625        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
626        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
627        json = (JSONObject) json.get(FILE_STATUSES_JSON);
628        JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON);
629        FileStatus[] array = new FileStatus[jsonArray.size()];
630        f = makeQualified(f);
631        for (int i = 0; i < jsonArray.size(); i++) {
632          array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i));
633        }
634        return array;
635      }
636    
637      /**
638       * Set the current working directory for the given file system. All relative
639       * paths will be resolved relative to it.
640       *
641       * @param newDir new directory.
642       */
643      @Override
644      public void setWorkingDirectory(Path newDir) {
645        workingDir = newDir;
646      }
647    
648      /**
649       * Get the current working directory for the given file system
650       *
651       * @return the directory pathname
652       */
653      @Override
654      public Path getWorkingDirectory() {
655        if (workingDir == null) {
656          workingDir = getHomeDirectory();
657        }
658        return workingDir;
659      }
660    
661      /**
662       * Make the given file and all non-existent parents into
663       * directories. Has the semantics of Unix 'mkdir -p'.
664       * Existence of the directory hierarchy is not an error.
665       */
666      @Override
667      public boolean mkdirs(Path f, FsPermission permission) throws IOException {
668        Map<String, String> params = new HashMap<String, String>();
669        params.put(OP_PARAM, Operation.MKDIRS.toString());
670        params.put(PERMISSION_PARAM, permissionToString(permission));
671        HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(),
672                                               params, f, true);
673        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
674        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
675        return (Boolean) json.get(MKDIRS_JSON);
676      }
677    
678      /**
679       * Return a file status object that represents the path.
680       *
681       * @param f The path we want information from
682       *
683       * @return a FileStatus object
684       *
685       * @throws FileNotFoundException when the path does not exist;
686       * IOException see specific implementation
687       */
688      @Override
689      public FileStatus getFileStatus(Path f) throws IOException {
690        Map<String, String> params = new HashMap<String, String>();
691        params.put(OP_PARAM, Operation.GETFILESTATUS.toString());
692        HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(),
693                                               params, f, true);
694        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
695        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
696        json = (JSONObject) json.get(FILE_STATUS_JSON);
697        f = makeQualified(f);
698        return createFileStatus(f, json);
699      }
700    
701      /**
702       * Return the current user's home directory in this filesystem.
703       * The default implementation returns "/user/$USER/".
704       */
705      @Override
706      public Path getHomeDirectory() {
707        Map<String, String> params = new HashMap<String, String>();
708        params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString());
709        try {
710          HttpURLConnection conn =
711            getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params,
712                          new Path(getUri().toString(), "/"), false);
713          HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
714          JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
715          return new Path((String) json.get(HOME_DIR_JSON));
716        } catch (IOException ex) {
717          throw new RuntimeException(ex);
718        }
719      }
720    
721      /**
722       * Set owner of a path (i.e. a file or a directory).
723       * The parameters username and groupname cannot both be null.
724       *
725       * @param p The path
726       * @param username If it is null, the original username remains unchanged.
727       * @param groupname If it is null, the original groupname remains unchanged.
728       */
729      @Override
730      public void setOwner(Path p, String username, String groupname)
731        throws IOException {
732        Map<String, String> params = new HashMap<String, String>();
733        params.put(OP_PARAM, Operation.SETOWNER.toString());
734        params.put(OWNER_PARAM, username);
735        params.put(GROUP_PARAM, groupname);
736        HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(),
737                                               params, p, true);
738        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
739      }
740    
741      /**
742       * Set permission of a path.
743       *
744       * @param p path.
745       * @param permission permission.
746       */
747      @Override
748      public void setPermission(Path p, FsPermission permission) throws IOException {
749        Map<String, String> params = new HashMap<String, String>();
750        params.put(OP_PARAM, Operation.SETPERMISSION.toString());
751        params.put(PERMISSION_PARAM, permissionToString(permission));
752        HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true);
753        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
754      }
755    
756      /**
757       * Set access time of a file
758       *
759       * @param p The path
760       * @param mtime Set the modification time of this file.
761       * The number of milliseconds since Jan 1, 1970.
762       * A value of -1 means that this call should not set modification time.
763       * @param atime Set the access time of this file.
764       * The number of milliseconds since Jan 1, 1970.
765       * A value of -1 means that this call should not set access time.
766       */
767      @Override
768      public void setTimes(Path p, long mtime, long atime) throws IOException {
769        Map<String, String> params = new HashMap<String, String>();
770        params.put(OP_PARAM, Operation.SETTIMES.toString());
771        params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime));
772        params.put(ACCESS_TIME_PARAM, Long.toString(atime));
773        HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(),
774                                               params, p, true);
775        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
776      }
777    
778      /**
779       * Set replication for an existing file.
780       *
781       * @param src file name
782       * @param replication new replication
783       *
784       * @return true if successful;
785       *         false if file does not exist or is a directory
786       *
787       * @throws IOException
788       */
789      @Override
790      public boolean setReplication(Path src, short replication)
791        throws IOException {
792        Map<String, String> params = new HashMap<String, String>();
793        params.put(OP_PARAM, Operation.SETREPLICATION.toString());
794        params.put(REPLICATION_PARAM, Short.toString(replication));
795        HttpURLConnection conn =
796          getConnection(Operation.SETREPLICATION.getMethod(), params, src, true);
797        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
798        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
799        return (Boolean) json.get(SET_REPLICATION_JSON);
800      }
801    
802      private FileStatus createFileStatus(Path parent, JSONObject json) {
803        String pathSuffix = (String) json.get(PATH_SUFFIX_JSON);
804        Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix);
805        FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON));
806        long len = (Long) json.get(LENGTH_JSON);
807        String owner = (String) json.get(OWNER_JSON);
808        String group = (String) json.get(GROUP_JSON);
809        FsPermission permission =
810          new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8));
811        long aTime = (Long) json.get(ACCESS_TIME_JSON);
812        long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
813        long blockSize = (Long) json.get(BLOCK_SIZE_JSON);
814        short replication = ((Long) json.get(REPLICATION_JSON)).shortValue();
815        FileStatus fileStatus = null;
816    
817        switch (type) {
818          case FILE:
819          case DIRECTORY:
820            fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY),
821                                        replication, blockSize, mTime, aTime,
822                                        permission, owner, group, path);
823            break;
824          case SYMLINK:
825            Path symLink = null;
826            fileStatus = new FileStatus(len, false,
827                                        replication, blockSize, mTime, aTime,
828                                        permission, owner, group, symLink,
829                                        path);
830        }
831        return fileStatus;
832      }
833    
834      @Override
835      public ContentSummary getContentSummary(Path f) throws IOException {
836        Map<String, String> params = new HashMap<String, String>();
837        params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString());
838        HttpURLConnection conn =
839          getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true);
840        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
841        JSONObject json = (JSONObject) ((JSONObject)
842          HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON);
843        return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON),
844                                  (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON),
845                                  (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON),
846                                  (Long) json.get(CONTENT_SUMMARY_QUOTA_JSON),
847                                  (Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON),
848                                  (Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON)
849        );
850      }
851    
852      @Override
853      public FileChecksum getFileChecksum(Path f) throws IOException {
854        Map<String, String> params = new HashMap<String, String>();
855        params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString());
856        HttpURLConnection conn =
857          getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true);
858        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
859        final JSONObject json = (JSONObject) ((JSONObject)
860          HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON);
861        return new FileChecksum() {
862          @Override
863          public String getAlgorithmName() {
864            return (String) json.get(CHECKSUM_ALGORITHM_JSON);
865          }
866    
867          @Override
868          public int getLength() {
869            return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue();
870          }
871    
872          @Override
873          public byte[] getBytes() {
874            return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON));
875          }
876    
877          @Override
878          public void write(DataOutput out) throws IOException {
879            throw new UnsupportedOperationException();
880          }
881    
882          @Override
883          public void readFields(DataInput in) throws IOException {
884            throw new UnsupportedOperationException();
885          }
886        };
887      }
888    
889    
890      @Override
891      public Token<?> getDelegationToken(final String renewer)
892        throws IOException {
893        return doAsRealUserIfNecessary(new Callable<Token<?>>() {
894          @Override
895          public Token<?> call() throws Exception {
896            return HttpFSKerberosAuthenticator.
897              getDelegationToken(uri, httpFSAddr, authToken, renewer);
898          }
899        });
900      }
901    
902      public long renewDelegationToken(final Token<?> token) throws IOException {
903        return doAsRealUserIfNecessary(new Callable<Long>() {
904          @Override
905          public Long call() throws Exception {
906            return HttpFSKerberosAuthenticator.
907              renewDelegationToken(uri,  authToken, token);
908          }
909        });
910      }
911    
912      public void cancelDelegationToken(final Token<?> token) throws IOException {
913        HttpFSKerberosAuthenticator.
914          cancelDelegationToken(uri, authToken, token);
915      }
916    
917      @Override
918      public Token<?> getRenewToken() {
919        return delegationToken;
920      }
921    
922      @Override
923      public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
924        delegationToken = token;
925      }
926    
927    }