View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.File;
21  import java.io.IOException;
22  import java.net.MalformedURLException;
23  import java.net.URL;
24  import java.util.HashMap;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileStatus;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  
34  /**
35   * This is a class loader that can load classes dynamically from new
36   * jar files under a configured folder. The paths to the jar files are
37   * converted to URLs, and URLClassLoader logic is actually used to load
38   * classes. This class loader always uses its parent class loader
39   * to load a class at first. Only if its parent class loader
40   * can not load a class, we will try to load it using the logic here.
41   * <p>
42   * The configured folder can be a HDFS path. In this case, the jar files
43   * under that folder will be copied to local at first under ${hbase.local.dir}/jars/.
44   * The local copy will be updated if the remote copy is updated, according to its
45   * last modified timestamp.
46   * <p>
47   * We can't unload a class already loaded. So we will use the existing
48   * jar files we already know to load any class which can't be loaded
49   * using the parent class loader. If we still can't load the class from
50   * the existing jar files, we will check if any new jar file is added,
51   * if so, we will load the new jar file and try to load the class again.
52   * If still failed, a class not found exception will be thrown.
53   * <p>
54   * Be careful in uploading new jar files and make sure all classes
55   * are consistent, otherwise, we may not be able to load your
56   * classes properly.
57   */
58  @InterfaceAudience.Private
59  public class DynamicClassLoader extends ClassLoaderBase {
60    private static final Log LOG =
61        LogFactory.getLog(DynamicClassLoader.class);
62  
63    // Dynamic jars are put under ${hbase.local.dir}/jars/
64    private static final String DYNAMIC_JARS_DIR = File.separator
65      + "jars" + File.separator;
66  
67    private static final String DYNAMIC_JARS_DIR_KEY = "hbase.dynamic.jars.dir";
68  
69    private static final String DYNAMIC_JARS_OPTIONAL_CONF_KEY = "hbase.use.dynamic.jars";
70    private static final boolean DYNAMIC_JARS_OPTIONAL_DEFAULT = true;
71  
72    private boolean useDynamicJars;
73  
74    private File localDir;
75  
76    // FileSystem of the remote path, set only if remoteDir != null
77    private FileSystem remoteDirFs;
78    private Path remoteDir;
79  
80    // Last modified time of local jars
81    private HashMap<String, Long> jarModifiedTime;
82  
83    /**
84     * Creates a DynamicClassLoader that can load classes dynamically
85     * from jar files under a specific folder.
86     *
87     * @param conf the configuration for the cluster.
88     * @param parent the parent ClassLoader to set.
89     */
90    public DynamicClassLoader(
91        final Configuration conf, final ClassLoader parent) {
92      super(parent);
93  
94      useDynamicJars = conf.getBoolean(
95          DYNAMIC_JARS_OPTIONAL_CONF_KEY, DYNAMIC_JARS_OPTIONAL_DEFAULT);
96  
97      if (useDynamicJars) {
98        initTempDir(conf);
99      }
100   }
101 
102   private void initTempDir(final Configuration conf) {
103     jarModifiedTime = new HashMap<String, Long>();
104     String localDirPath = conf.get(
105       LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + DYNAMIC_JARS_DIR;
106     localDir = new File(localDirPath);
107     if (!localDir.mkdirs() && !localDir.isDirectory()) {
108       throw new RuntimeException("Failed to create local dir " + localDir.getPath()
109         + ", DynamicClassLoader failed to init");
110     }
111 
112     String remotePath = conf.get(DYNAMIC_JARS_DIR_KEY);
113     if (remotePath == null || remotePath.equals(localDirPath)) {
114       remoteDir = null;  // ignore if it is the same as the local path
115     } else {
116       remoteDir = new Path(remotePath);
117       try {
118         remoteDirFs = remoteDir.getFileSystem(conf);
119       } catch (IOException ioe) {
120         LOG.warn("Failed to identify the fs of dir "
121           + remoteDir + ", ignored", ioe);
122         remoteDir = null;
123       }
124     }
125   }
126 
127   @Override
128   public Class<?> loadClass(String name)
129       throws ClassNotFoundException {
130     try {
131       return parent.loadClass(name);
132     } catch (ClassNotFoundException e) {
133       if (LOG.isDebugEnabled()) {
134         LOG.debug("Class " + name + " not found - using dynamical class loader");
135       }
136 
137       if (useDynamicJars) {
138         return tryRefreshClass(name);
139       }
140       throw e;
141     }
142   }
143 
144 
145   private Class<?> tryRefreshClass(String name)
146       throws ClassNotFoundException {
147     synchronized (getClassLoadingLock(name)) {
148         // Check whether the class has already been loaded:
149         Class<?> clasz = findLoadedClass(name);
150         if (clasz != null) {
151           if (LOG.isDebugEnabled()) {
152             LOG.debug("Class " + name + " already loaded");
153           }
154         }
155         else {
156           try {
157             if (LOG.isDebugEnabled()) {
158               LOG.debug("Finding class: " + name);
159             }
160             clasz = findClass(name);
161           } catch (ClassNotFoundException cnfe) {
162             // Load new jar files if any
163             if (LOG.isDebugEnabled()) {
164               LOG.debug("Loading new jar files, if any");
165             }
166             loadNewJars();
167 
168             if (LOG.isDebugEnabled()) {
169               LOG.debug("Finding class again: " + name);
170             }
171             clasz = findClass(name);
172           }
173         }
174         return clasz;
175       }
176   }
177 
178   private synchronized void loadNewJars() {
179     // Refresh local jar file lists
180     if (localDir != null) {
181       for (File file : localDir.listFiles()) {
182         String fileName = file.getName();
183         if (jarModifiedTime.containsKey(fileName)) {
184           continue;
185         }
186         if (file.isFile() && fileName.endsWith(".jar")) {
187           jarModifiedTime.put(fileName, Long.valueOf(file.lastModified()));
188           try {
189             URL url = file.toURI().toURL();
190             addURL(url);
191           } catch (MalformedURLException mue) {
192             // This should not happen, just log it
193             LOG.warn("Failed to load new jar " + fileName, mue);
194           }
195         }
196       }
197     }
198 
199     // Check remote files
200     FileStatus[] statuses = null;
201     if (remoteDir != null) {
202       try {
203         statuses = remoteDirFs.listStatus(remoteDir);
204       } catch (IOException ioe) {
205         LOG.warn("Failed to check remote dir status " + remoteDir, ioe);
206       }
207     }
208     if (statuses == null || statuses.length == 0) {
209       return; // no remote files at all
210     }
211 
212     for (FileStatus status: statuses) {
213       if (status.isDir()) continue; // No recursive lookup
214       Path path = status.getPath();
215       String fileName = path.getName();
216       if (!fileName.endsWith(".jar")) {
217         if (LOG.isDebugEnabled()) {
218           LOG.debug("Ignored non-jar file " + fileName);
219         }
220         continue; // Ignore non-jar files
221       }
222       Long cachedLastModificationTime = jarModifiedTime.get(fileName);
223       if (cachedLastModificationTime != null) {
224         long lastModified = status.getModificationTime();
225         if (lastModified < cachedLastModificationTime.longValue()) {
226           // There could be some race, for example, someone uploads
227           // a new one right in the middle the old one is copied to
228           // local. We can check the size as well. But it is still
229           // not guaranteed. This should be rare. Most likely,
230           // we already have the latest one.
231           // If you are unlucky to hit this race issue, you have
232           // to touch the remote jar to update its last modified time
233           continue;
234         }
235       }
236       try {
237         // Copy it to local
238         File dst = new File(localDir, fileName);
239         remoteDirFs.copyToLocalFile(path, new Path(dst.getPath()));
240         jarModifiedTime.put(fileName, Long.valueOf(dst.lastModified()));
241         URL url = dst.toURI().toURL();
242         addURL(url);
243       } catch (IOException ioe) {
244         LOG.warn("Failed to load new jar " + fileName, ioe);
245       }
246     }
247   }
248 }