View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.File;
21  import java.io.IOException;
22  import java.net.MalformedURLException;
23  import java.net.URL;
24  import java.net.URLClassLoader;
25  import java.util.HashMap;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.classification.InterfaceAudience;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileStatus;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  
35  /**
36   * This is a class loader that can load classes dynamically from new
37   * jar files under a configured folder. It always uses its parent class
38   * loader to load a class at first. Only if its parent class loader
39   * can not load a class, we will try to load it using the logic here.
40   * <p>
41   * We can't unload a class already loaded. So we will use the existing
42   * jar files we already know to load any class which can't be loaded
43   * using the parent class loader. If we still can't load the class from
44   * the existing jar files, we will check if any new jar file is added,
45   * if so, we will load the new jar file and try to load the class again.
46   * If still failed, a class not found exception will be thrown.
47   * <p>
48   * Be careful in uploading new jar files and make sure all classes
49   * are consistent, otherwise, we may not be able to load your
50   * classes properly.
51   */
52  @InterfaceAudience.Private
53  public class DynamicClassLoader extends URLClassLoader {
54    private static final Log LOG =
55        LogFactory.getLog(DynamicClassLoader.class);
56  
57    // Dynamic jars are put under ${hbase.local.dir}/dynamic/jars/
58    private static final String DYNAMIC_JARS_DIR = File.separator
59      + "dynamic" + File.separator + "jars" + File.separator;
60  
61    /**
62     * Parent class loader used to load any class at first.
63     */
64    private final ClassLoader parent;
65  
66    private File localDir;
67  
68    // FileSystem of the remote path, set only if remoteDir != null
69    private FileSystem remoteDirFs;
70    private Path remoteDir;
71  
72    // Last modified time of local jars
73    private HashMap<String, Long> jarModifiedTime;
74  
75    /**
76     * Creates a DynamicClassLoader that can load classes dynamically
77     * from jar files under a specific folder.
78     *
79     * @param conf the configuration for the cluster.
80     * @param parent the parent ClassLoader to set.
81     */
82    public DynamicClassLoader(
83        final Configuration conf, final ClassLoader parent) {
84      super(new URL[]{}, parent);
85      this.parent = parent;
86  
87      jarModifiedTime = new HashMap<String, Long>();
88      String localDirPath = conf.get("hbase.local.dir") + DYNAMIC_JARS_DIR;
89      localDir = new File(localDirPath);
90      if (!localDir.mkdirs() && !localDir.isDirectory()) {
91        throw new RuntimeException("Failed to create local dir " + localDir.getPath()
92          + ", DynamicClassLoader failed to init");
93      }
94  
95      String remotePath = conf.get("hbase.dynamic.jars.dir");
96      if (remotePath == null || remotePath.equals(localDirPath)) {
97        remoteDir = null;  // ignore if it is the same as the local path
98      } else {
99        remoteDir = new Path(remotePath);
100       try {
101         remoteDirFs = remoteDir.getFileSystem(conf);
102       } catch (IOException ioe) {
103         LOG.warn("Failed to identify the fs of dir "
104           + remoteDir + ", ignored", ioe);
105         remoteDir = null;
106       }
107     }
108   }
109 
110   @Override
111   public Class<?> loadClass(String name)
112       throws ClassNotFoundException {
113     try {
114       return parent.loadClass(name);
115     } catch (ClassNotFoundException e) {
116       if (LOG.isDebugEnabled()) {
117         LOG.debug("Class " + name + " not found - using dynamical class loader");
118       }
119 
120       // Check whether the class has already been loaded:
121       Class<?> clasz = findLoadedClass(name);
122       if (clasz != null) {
123         if (LOG.isDebugEnabled()) {
124           LOG.debug("Class " + name + " already loaded");
125         }
126       }
127       else {
128         try {
129           if (LOG.isDebugEnabled()) {
130             LOG.debug("Finding class: " + name);
131           }
132           clasz = findClass(name);
133         } catch (ClassNotFoundException cnfe) {
134           // Load new jar files if any
135           if (LOG.isDebugEnabled()) {
136             LOG.debug("Loading new jar files, if any");
137           }
138           loadNewJars();
139 
140           if (LOG.isDebugEnabled()) {
141             LOG.debug("Finding class again: " + name);
142           }
143           clasz = findClass(name);
144         }
145       }
146       return clasz;
147     }
148   }
149 
150   private synchronized void loadNewJars() {
151     // Refresh local jar file lists
152     for (File file: localDir.listFiles()) {
153       String fileName = file.getName();
154       if (jarModifiedTime.containsKey(fileName)) {
155         continue;
156       }
157       if (file.isFile() && fileName.endsWith(".jar")) {
158         jarModifiedTime.put(fileName, Long.valueOf(file.lastModified()));
159         try {
160           URL url = file.toURI().toURL();
161           addURL(url);
162         } catch (MalformedURLException mue) {
163           // This should not happen, just log it
164           LOG.warn("Failed to load new jar " + fileName, mue);
165         }
166       }
167     }
168 
169     // Check remote files
170     FileStatus[] statuses = null;
171     if (remoteDir != null) {
172       try {
173         statuses = remoteDirFs.listStatus(remoteDir);
174       } catch (IOException ioe) {
175         LOG.warn("Failed to check remote dir status " + remoteDir, ioe);
176       }
177     }
178     if (statuses == null || statuses.length == 0) {
179       return; // no remote files at all
180     }
181 
182     for (FileStatus status: statuses) {
183       if (status.isDir()) continue; // No recursive lookup
184       Path path = status.getPath();
185       String fileName = path.getName();
186       if (!fileName.endsWith(".jar")) {
187         if (LOG.isDebugEnabled()) {
188           LOG.debug("Ignored non-jar file " + fileName);
189         }
190         continue; // Ignore non-jar files
191       }
192       Long cachedLastModificationTime = jarModifiedTime.get(fileName);
193       if (cachedLastModificationTime != null) {
194         long lastModified = status.getModificationTime();
195         if (lastModified < cachedLastModificationTime.longValue()) {
196           // There could be some race, for example, someone uploads
197           // a new one right in the middle the old one is copied to
198           // local. We can check the size as well. But it is still
199           // not guaranteed. This should be rare. Most likely,
200           // we already have the latest one.
201           // If you are unlucky to hit this race issue, you have
202           // to touch the remote jar to update its last modified time
203           continue;
204         }
205       }
206       try {
207         // Copy it to local
208         File dst = new File(localDir, fileName);
209         remoteDirFs.copyToLocalFile(path, new Path(dst.getPath()));
210         jarModifiedTime.put(fileName, Long.valueOf(dst.lastModified()));
211         URL url = dst.toURI().toURL();
212         addURL(url);
213       } catch (IOException ioe) {
214         LOG.warn("Failed to load new jar " + fileName, ioe);
215       }
216     }
217   }
218 }