View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.zookeeper;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HBaseConfiguration;
26  import org.apache.hadoop.hbase.HConstants;
27  import org.apache.hadoop.net.DNS;
28  import org.apache.hadoop.util.StringUtils;
29  import org.apache.zookeeper.server.ServerConfig;
30  import org.apache.zookeeper.server.ZooKeeperServerMain;
31  import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
32  import org.apache.zookeeper.server.quorum.QuorumPeerMain;
33  
34  import java.io.File;
35  import java.io.IOException;
36  import java.io.InputStream;
37  import java.io.PrintWriter;
38  import java.net.InetAddress;
39  import java.net.NetworkInterface;
40  import java.net.UnknownHostException;
41  import java.util.ArrayList;
42  import java.util.Enumeration;
43  import java.util.List;
44  import java.util.Map.Entry;
45  import java.util.Properties;
46  
47  /**
48   * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage
49   * ZooKeeper, this class is used to start up QuorumPeer instances. By doing
50   * things in here rather than directly calling to ZooKeeper, we have more
51   * control over the process. Currently, this class allows us to parse the
52   * zoo.cfg and inject variables from HBase's site.xml configuration in.
53   */
54  public class HQuorumPeer {
55    private static final Log LOG = LogFactory.getLog(HQuorumPeer.class);
56  
57    private static final String VARIABLE_START = "${";
58    private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
59    private static final String VARIABLE_END = "}";
60    private static final int VARIABLE_END_LENGTH = VARIABLE_END.length();
61  
62    private static final String ZK_CFG_PROPERTY = "hbase.zookeeper.property.";
63    private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
64    private static final String ZK_CLIENT_PORT_KEY = ZK_CFG_PROPERTY
65        + "clientPort";
66  
67    /**
68     * Parse ZooKeeper configuration from HBase XML config and run a QuorumPeer.
69     * @param args String[] of command line arguments. Not used.
70     */
71    public static void main(String[] args) {
72      Configuration conf = HBaseConfiguration.create();
73      try {
74        Properties zkProperties = makeZKProps(conf);
75        writeMyID(zkProperties);
76        QuorumPeerConfig zkConfig = new QuorumPeerConfig();
77        zkConfig.parseProperties(zkProperties);
78        runZKServer(zkConfig);
79      } catch (Exception e) {
80        e.printStackTrace();
81        System.exit(-1);
82      }
83    }
84  
85    private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException {
86      if (zkConfig.isDistributed()) {
87        QuorumPeerMain qp = new QuorumPeerMain();
88        qp.runFromConfig(zkConfig);
89      } else {
90        ZooKeeperServerMain zk = new ZooKeeperServerMain();
91        ServerConfig serverConfig = new ServerConfig();
92        serverConfig.readFrom(zkConfig);
93        zk.runFromConfig(serverConfig);
94      }
95    }
96  
97    private static boolean addressIsLocalHost(String address) {
98      return address.equals("localhost") || address.equals("127.0.0.1");
99    }
100 
101   private static void writeMyID(Properties properties) throws IOException {
102     long myId = -1;
103 
104     Configuration conf = HBaseConfiguration.create();
105     String myAddress = DNS.getDefaultHost(
106         conf.get("hbase.zookeeper.dns.interface","default"),
107         conf.get("hbase.zookeeper.dns.nameserver","default"));
108 
109     List<String> ips = new ArrayList<String>();
110 
111     // Add what could be the best (configured) match
112     ips.add(myAddress.contains(".") ?
113         myAddress :
114         StringUtils.simpleHostname(myAddress));
115 
116     // For all nics get all hostnames and IPs
117     Enumeration<?> nics = NetworkInterface.getNetworkInterfaces();
118     while(nics.hasMoreElements()) {
119       Enumeration<?> rawAdrs =
120           ((NetworkInterface)nics.nextElement()).getInetAddresses();
121       while(rawAdrs.hasMoreElements()) {
122         InetAddress inet = (InetAddress) rawAdrs.nextElement();
123         ips.add(StringUtils.simpleHostname(inet.getHostName()));
124         ips.add(inet.getHostAddress());
125       }
126     }
127 
128     for (Entry<Object, Object> entry : properties.entrySet()) {
129       String key = entry.getKey().toString().trim();
130       String value = entry.getValue().toString().trim();
131       if (key.startsWith("server.")) {
132         int dot = key.indexOf('.');
133         long id = Long.parseLong(key.substring(dot + 1));
134         String[] parts = value.split(":");
135         String address = parts[0];
136         if (addressIsLocalHost(address) || ips.contains(address)) {
137           myId = id;
138           break;
139         }
140       }
141     }
142 
143     if (myId == -1) {
144       throw new IOException("Could not find my address: " + myAddress +
145                             " in list of ZooKeeper quorum servers");
146     }
147 
148     String dataDirStr = properties.get("dataDir").toString().trim();
149     File dataDir = new File(dataDirStr);
150     if (!dataDir.isDirectory()) {
151       if (!dataDir.mkdirs()) {
152         throw new IOException("Unable to create data dir " + dataDir);
153       }
154     }
155 
156     File myIdFile = new File(dataDir, "myid");
157     PrintWriter w = new PrintWriter(myIdFile);
158     w.println(myId);
159     w.close();
160   }
161 
162   /**
163    * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
164    * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
165    * the corresponding config options from the HBase XML configs and generate
166    * the appropriate ZooKeeper properties.
167    * @param conf Configuration to read from.
168    * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
169    */
170   public static Properties makeZKProps(Configuration conf) {
171     // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
172     // it and grab its configuration properties.
173     ClassLoader cl = HQuorumPeer.class.getClassLoader();
174     final InputStream inputStream =
175       cl.getResourceAsStream(HConstants.ZOOKEEPER_CONFIG_NAME);
176     if (inputStream != null) {
177       try {
178         return parseZooCfg(conf, inputStream);
179       } catch (IOException e) {
180         LOG.warn("Cannot read " + HConstants.ZOOKEEPER_CONFIG_NAME +
181                  ", loading from XML files", e);
182       }
183     }
184 
185     // Otherwise, use the configuration options from HBase's XML files.
186     Properties zkProperties = new Properties();
187 
188     // Directly map all of the hbase.zookeeper.property.KEY properties.
189     for (Entry<String, String> entry : conf) {
190       String key = entry.getKey();
191       if (key.startsWith(ZK_CFG_PROPERTY)) {
192         String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE);
193         String value = entry.getValue();
194         // If the value has variables substitutions, need to do a get.
195         if (value.contains(VARIABLE_START)) {
196           value = conf.get(key);
197         }
198         zkProperties.put(zkKey, value);
199       }
200     }
201 
202     // If clientPort is not set, assign the default
203     if (zkProperties.getProperty(ZK_CLIENT_PORT_KEY) == null) {
204       zkProperties.put(ZK_CLIENT_PORT_KEY,
205                        HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
206     }
207 
208     // Create the server.X properties.
209     int peerPort = conf.getInt("hbase.zookeeper.peerport", 2888);
210     int leaderPort = conf.getInt("hbase.zookeeper.leaderport", 3888);
211 
212     final String[] serverHosts = conf.getStrings(HConstants.ZOOKEEPER_QUORUM,
213                                                  "localhost");
214     for (int i = 0; i < serverHosts.length; ++i) {
215       String serverHost = serverHosts[i];
216       String address = serverHost + ":" + peerPort + ":" + leaderPort;
217       String key = "server." + i;
218       zkProperties.put(key, address);
219     }
220 
221     return zkProperties;
222   }
223   
224   /**
225    * Return the ZK Quorum servers string given zk properties returned by 
226    * makeZKProps
227    * @param properties
228    * @return
229    */
230   public static String getZKQuorumServersString(Properties properties) {
231     String clientPort = null;
232     List<String> servers = new ArrayList<String>();
233 
234     // The clientPort option may come after the server.X hosts, so we need to
235     // grab everything and then create the final host:port comma separated list.
236     boolean anyValid = false;
237     for (Entry<Object,Object> property : properties.entrySet()) {
238       String key = property.getKey().toString().trim();
239       String value = property.getValue().toString().trim();
240       if (key.equals("clientPort")) {
241         clientPort = value;
242       }
243       else if (key.startsWith("server.")) {
244         String host = value.substring(0, value.indexOf(':'));
245         servers.add(host);
246         try {
247           //noinspection ResultOfMethodCallIgnored
248           InetAddress.getByName(host);
249           anyValid = true;
250         } catch (UnknownHostException e) {
251           LOG.warn(StringUtils.stringifyException(e));
252         }
253       }
254     }
255 
256     if (!anyValid) {
257       LOG.error("no valid quorum servers found in " + HConstants.ZOOKEEPER_CONFIG_NAME);
258       return null;
259     }
260 
261     if (clientPort == null) {
262       LOG.error("no clientPort found in " + HConstants.ZOOKEEPER_CONFIG_NAME);
263       return null;
264     }
265 
266     if (servers.isEmpty()) {
267       LOG.fatal("No server.X lines found in conf/zoo.cfg. HBase must have a " +
268                 "ZooKeeper cluster configured for its operation.");
269       return null;
270     }
271 
272     StringBuilder hostPortBuilder = new StringBuilder();
273     for (int i = 0; i < servers.size(); ++i) {
274       String host = servers.get(i);
275       if (i > 0) {
276         hostPortBuilder.append(',');
277       }
278       hostPortBuilder.append(host);
279       hostPortBuilder.append(':');
280       hostPortBuilder.append(clientPort);
281     }
282 
283     return hostPortBuilder.toString();
284   }
285 
286   /**
287    * Parse ZooKeeper's zoo.cfg, injecting HBase Configuration variables in.
288    * This method is used for testing so we can pass our own InputStream.
289    * @param conf HBaseConfiguration to use for injecting variables.
290    * @param inputStream InputStream to read from.
291    * @return Properties parsed from config stream with variables substituted.
292    * @throws IOException if anything goes wrong parsing config
293    */
294   public static Properties parseZooCfg(Configuration conf,
295       InputStream inputStream) throws IOException {
296     Properties properties = new Properties();
297     try {
298       properties.load(inputStream);
299     } catch (IOException e) {
300       final String msg = "fail to read properties from "
301         + HConstants.ZOOKEEPER_CONFIG_NAME;
302       LOG.fatal(msg);
303       throw new IOException(msg, e);
304     }
305     for (Entry<Object, Object> entry : properties.entrySet()) {
306       String value = entry.getValue().toString().trim();
307       String key = entry.getKey().toString().trim();
308       StringBuilder newValue = new StringBuilder();
309       int varStart = value.indexOf(VARIABLE_START);
310       int varEnd = 0;
311       while (varStart != -1) {
312         varEnd = value.indexOf(VARIABLE_END, varStart);
313         if (varEnd == -1) {
314           String msg = "variable at " + varStart + " has no end marker";
315           LOG.fatal(msg);
316           throw new IOException(msg);
317         }
318         String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd);
319 
320         String substituteValue = System.getProperty(variable);
321         if (substituteValue == null) {
322           substituteValue = conf.get(variable);
323         }
324         if (substituteValue == null) {
325           String msg = "variable " + variable + " not set in system property "
326                      + "or hbase configs";
327           LOG.fatal(msg);
328           throw new IOException(msg);
329         }
330 
331         newValue.append(substituteValue);
332 
333         varEnd += VARIABLE_END_LENGTH;
334         varStart = value.indexOf(VARIABLE_START, varEnd);
335       }
336       // Special case for 'hbase.cluster.distributed' property being 'true'
337       if (key.startsWith("server.")) {
338         if (conf.get(HConstants.CLUSTER_DISTRIBUTED).equals(HConstants.CLUSTER_IS_DISTRIBUTED)
339             && value.startsWith("localhost")) {
340           String msg = "The server in zoo.cfg cannot be set to localhost " +
341               "in a fully-distributed setup because it won't be reachable. " +
342               "See \"Getting Started\" for more information.";
343           LOG.fatal(msg);
344           throw new IOException(msg);
345         }
346       }
347       newValue.append(value.substring(varEnd));
348       properties.setProperty(key, newValue.toString());
349     }
350     return properties;
351   }
352 }