View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import java.io.IOException;
21  import java.lang.reflect.InvocationTargetException;
22  import java.lang.reflect.Method;
23  import java.util.Map;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.hbase.classification.InterfaceStability;
30  import org.apache.hadoop.hbase.util.VersionInfo;
31  import org.apache.hadoop.hbase.zookeeper.ZKConfig;
32  
33  /**
34   * Adds HBase configuration files to a Configuration
35   */
36  @InterfaceAudience.Public
37  @InterfaceStability.Stable
38  public class HBaseConfiguration extends Configuration {
39  
40    private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class);
41  
42    // a constant to convert a fraction to a percentage
43    private static final int CONVERT_TO_PERCENTAGE = 100;
44  
45    /**
46     * Instantinating HBaseConfiguration() is deprecated. Please use
47     * HBaseConfiguration#create() to construct a plain Configuration
48     */
49    @Deprecated
50    public HBaseConfiguration() {
51      //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration
52      super();
53      addHbaseResources(this);
54      LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use"
55          + " HBaseConfiguration#create() to construct a plain Configuration");
56    }
57  
58    /**
59     * Instantiating HBaseConfiguration() is deprecated. Please use
60     * HBaseConfiguration#create(conf) to construct a plain Configuration
61     */
62    @Deprecated
63    public HBaseConfiguration(final Configuration c) {
64      //TODO:replace with private constructor
65      this();
66      merge(this, c);
67    }
68  
69    private static void checkDefaultsVersion(Configuration conf) {
70      if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return;
71      String defaultsVersion = conf.get("hbase.defaults.for.version");
72      String thisVersion = VersionInfo.getVersion();
73      if (!thisVersion.equals(defaultsVersion)) {
74        throw new RuntimeException(
75          "hbase-default.xml file seems to be for and old version of HBase (" +
76          defaultsVersion + "), this version is " + thisVersion);
77      }
78    }
79  
80    private static void checkForClusterFreeMemoryLimit(Configuration conf) {
81        float globalMemstoreLimit = conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f);
82        int gml = (int)(globalMemstoreLimit * CONVERT_TO_PERCENTAGE);
83        float blockCacheUpperLimit =
84          conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
85            HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
86        int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
87        if (CONVERT_TO_PERCENTAGE - (gml + bcul)
88                < (int)(CONVERT_TO_PERCENTAGE *
89                        HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) {
90            throw new RuntimeException(
91              "Current heap configuration for MemStore and BlockCache exceeds " +
92              "the threshold required for successful cluster operation. " +
93              "The combined value cannot exceed 0.8. Please check " +
94              "the settings for hbase.regionserver.global.memstore.upperLimit and " +
95              "hfile.block.cache.size in your configuration. " +
96              "hbase.regionserver.global.memstore.upperLimit is " +
97              globalMemstoreLimit +
98              " hfile.block.cache.size is " + blockCacheUpperLimit);
99        }
100   }
101 
102   public static Configuration addHbaseResources(Configuration conf) {
103     conf.addResource("hbase-default.xml");
104     conf.addResource("hbase-site.xml");
105 
106     checkDefaultsVersion(conf);
107     checkForClusterFreeMemoryLimit(conf);
108     return conf;
109   }
110 
111   /**
112    * Creates a Configuration with HBase resources
113    * @return a Configuration with HBase resources
114    */
115   public static Configuration create() {
116     Configuration conf = new Configuration();
117     // In case HBaseConfiguration is loaded from a different classloader than
118     // Configuration, conf needs to be set with appropriate class loader to resolve
119     // HBase resources.
120     conf.setClassLoader(HBaseConfiguration.class.getClassLoader());
121     return addHbaseResources(conf);
122   }
123 
124   /**
125    * @param that Configuration to clone.
126    * @return a Configuration created with the hbase-*.xml files plus
127    * the given configuration.
128    */
129   public static Configuration create(final Configuration that) {
130     Configuration conf = create();
131     merge(conf, that);
132     return conf;
133   }
134 
135   /**
136    * Merge two configurations.
137    * @param destConf the configuration that will be overwritten with items
138    *                 from the srcConf
139    * @param srcConf the source configuration
140    **/
141   public static void merge(Configuration destConf, Configuration srcConf) {
142     for (Map.Entry<String, String> e : srcConf) {
143       destConf.set(e.getKey(), e.getValue());
144     }
145   }
146 
147   /**
148    * Returns a subset of the configuration properties, matching the given key prefix.
149    * The prefix is stripped from the return keys, ie. when calling with a prefix of "myprefix",
150    * the entry "myprefix.key1 = value1" would be returned as "key1 = value1".  If an entry's
151    * key matches the prefix exactly ("myprefix = value2"), it will <strong>not</strong> be
152    * included in the results, since it would show up as an entry with an empty key.
153    */
154   public static Configuration subset(Configuration srcConf, String prefix) {
155     Configuration newConf = new Configuration(false);
156     for (Map.Entry<String, String> entry : srcConf) {
157       if (entry.getKey().startsWith(prefix)) {
158         String newKey = entry.getKey().substring(prefix.length());
159         // avoid entries that would produce an empty key
160         if (!newKey.isEmpty()) {
161           newConf.set(newKey, entry.getValue());
162         }
163       }
164     }
165     return newConf;
166   }
167 
168   /**
169    * Sets all the entries in the provided {@code Map<String, String>} as properties in the
170    * given {@code Configuration}.  Each property will have the specified prefix prepended,
171    * so that the configuration entries are keyed by {@code prefix + entry.getKey()}.
172    */
173   public static void setWithPrefix(Configuration conf, String prefix,
174                                    Iterable<Map.Entry<String, String>> properties) {
175     for (Map.Entry<String, String> entry : properties) {
176       conf.set(prefix + entry.getKey(), entry.getValue());
177     }
178   }
179 
180   /**
181    * @return whether to show HBase Configuration in servlet
182    */
183   public static boolean isShowConfInServlet() {
184     boolean isShowConf = false;
185     try {
186       if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) {
187         isShowConf = true;
188       }
189     } catch (LinkageError e) {
190        // should we handle it more aggressively in addition to log the error?
191        LOG.warn("Error thrown: ", e);
192     } catch (ClassNotFoundException ce) {
193       LOG.debug("ClassNotFound: ConfServlet");
194       // ignore
195     }
196     return isShowConf;
197   }
198 
199   /**
200    * Get the value of the <code>name</code> property as an <code>int</code>, possibly
201    * referring to the deprecated name of the configuration property.
202    * If no such property exists, the provided default value is returned,
203    * or if the specified value is not a valid <code>int</code>,
204    * then an error is thrown.
205    *
206    * @param name property name.
207    * @param deprecatedName a deprecatedName for the property to use
208    * if non-deprecated name is not used
209    * @param defaultValue default value.
210    * @throws NumberFormatException when the value is invalid
211    * @return property value as an <code>int</code>,
212    *         or <code>defaultValue</code>.
213    */
214   // TODO: developer note: This duplicates the functionality of deprecated
215   // property support in Configuration in Hadoop 2. But since Hadoop-1 does not
216   // contain these changes, we will do our own as usual. Replace these when H2 is default.
217   public static int getInt(Configuration conf, String name,
218       String deprecatedName, int defaultValue) {
219     if (conf.get(deprecatedName) != null) {
220       LOG.warn(String.format("Config option \"%s\" is deprecated. Instead, use \"%s\""
221         , deprecatedName, name));
222       return conf.getInt(deprecatedName, defaultValue);
223     } else {
224       return conf.getInt(name, defaultValue);
225     }
226   }
227 
228   /**
229    * Get the password from the Configuration instance using the
230    * getPassword method if it exists. If not, then fall back to the
231    * general get method for configuration elements.
232    * @param conf configuration instance for accessing the passwords
233    * @param alias the name of the password element
234    * @param defPass the default password
235    * @return String password or default password
236    * @throws IOException
237    */
238   public static String getPassword(Configuration conf, String alias,
239       String defPass) throws IOException {
240     String passwd = null;
241     try {
242       Method m = Configuration.class.getMethod("getPassword", String.class);
243       char[] p = (char[]) m.invoke(conf, alias);
244       if (p != null) {
245         LOG.debug(String.format("Config option \"%s\" was found through" +
246         		" the Configuration getPassword method.", alias));
247         passwd = new String(p);
248       }
249       else {
250         LOG.debug(String.format(
251             "Config option \"%s\" was not found. Using provided default value",
252             alias));
253         passwd = defPass;
254       }
255     } catch (NoSuchMethodException e) {
256       // this is a version of Hadoop where the credential
257       //provider API doesn't exist yet
258       LOG.debug(String.format(
259           "Credential.getPassword method is not available." +
260           " Falling back to configuration."));
261       passwd = conf.get(alias, defPass);
262     } catch (SecurityException e) {
263       throw new IOException(e.getMessage(), e);
264     } catch (IllegalAccessException e) {
265       throw new IOException(e.getMessage(), e);
266     } catch (IllegalArgumentException e) {
267       throw new IOException(e.getMessage(), e);
268     } catch (InvocationTargetException e) {
269       throw new IOException(e.getMessage(), e);
270     }
271     return passwd;
272   }
273 
274   /**
275    * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key
276    * to the base Configuration.  Note that additional configuration properties may be needed
277    * for a remote cluster, so it is preferable to use
278    * {@link #createClusterConf(Configuration, String, String)}.
279    *
280    * @param baseConf the base configuration to use, containing prefixed override properties
281    * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none
282    *
283    * @return the merged configuration with override properties and cluster key applied
284    *
285    * @see #createClusterConf(Configuration, String, String)
286    */
287   public static Configuration createClusterConf(Configuration baseConf, String clusterKey)
288       throws IOException {
289     return createClusterConf(baseConf, clusterKey, null);
290   }
291 
292   /**
293    * Generates a {@link Configuration} instance by applying property overrides prefixed by
294    * a cluster profile key to the base Configuration.  Override properties are extracted by
295    * the {@link #subset(Configuration, String)} method, then the merged on top of the base
296    * Configuration and returned.
297    *
298    * @param baseConf the base configuration to use, containing prefixed override properties
299    * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none
300    * @param overridePrefix the property key prefix to match for override properties,
301    *     or {@code null} if none
302    * @return the merged configuration with override properties and cluster key applied
303    */
304   public static Configuration createClusterConf(Configuration baseConf, String clusterKey,
305                                                 String overridePrefix) throws IOException {
306     Configuration clusterConf = HBaseConfiguration.create(baseConf);
307     if (clusterKey != null && !clusterKey.isEmpty()) {
308       applyClusterKeyToConf(clusterConf, clusterKey);
309     }
310 
311     if (overridePrefix != null && !overridePrefix.isEmpty()) {
312       Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix);
313       HBaseConfiguration.merge(clusterConf, clusterSubset);
314     }
315     return clusterConf;
316   }
317 
318   /**
319    * Apply the settings in the given key to the given configuration, this is
320    * used to communicate with distant clusters
321    * @param conf configuration object to configure
322    * @param key string that contains the 3 required configuratins
323    * @throws IOException
324    */
325   private static void applyClusterKeyToConf(Configuration conf, String key)
326       throws IOException{
327     ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key);
328     conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString());
329     conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort());
330     conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent());
331   }
332 
333   /**
334    * For debugging.  Dump configurations to system output as xml format.
335    * Master and RS configurations can also be dumped using
336    * http services. e.g. "curl http://master:60010/dump"
337    */
338   public static void main(String[] args) throws Exception {
339     HBaseConfiguration.create().writeXml(System.out);
340   }
341 }