1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.zookeeper;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.HBaseConfiguration;
26 import org.apache.hadoop.hbase.HConstants;
27 import org.apache.hadoop.net.DNS;
28 import org.apache.hadoop.util.StringUtils;
29 import org.apache.zookeeper.server.ServerConfig;
30 import org.apache.zookeeper.server.ZooKeeperServerMain;
31 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
32 import org.apache.zookeeper.server.quorum.QuorumPeerMain;
33
34 import java.io.File;
35 import java.io.IOException;
36 import java.io.InputStream;
37 import java.io.PrintWriter;
38 import java.net.InetAddress;
39 import java.net.NetworkInterface;
40 import java.net.UnknownHostException;
41 import java.util.ArrayList;
42 import java.util.Enumeration;
43 import java.util.List;
44 import java.util.Map.Entry;
45 import java.util.Properties;
46
47
48
49
50
51
52
53
54 public class HQuorumPeer {
55 private static final Log LOG = LogFactory.getLog(HQuorumPeer.class);
56
57 private static final String VARIABLE_START = "${";
58 private static final int VARIABLE_START_LENGTH = VARIABLE_START.length();
59 private static final String VARIABLE_END = "}";
60 private static final int VARIABLE_END_LENGTH = VARIABLE_END.length();
61
62 private static final String ZK_CFG_PROPERTY = "hbase.zookeeper.property.";
63 private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
64 private static final String ZK_CLIENT_PORT_KEY = ZK_CFG_PROPERTY
65 + "clientPort";
66
67
68
69
70
71 public static void main(String[] args) {
72 Configuration conf = HBaseConfiguration.create();
73 try {
74 Properties zkProperties = makeZKProps(conf);
75 writeMyID(zkProperties);
76 QuorumPeerConfig zkConfig = new QuorumPeerConfig();
77 zkConfig.parseProperties(zkProperties);
78 runZKServer(zkConfig);
79 } catch (Exception e) {
80 e.printStackTrace();
81 System.exit(-1);
82 }
83 }
84
85 private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException {
86 if (zkConfig.isDistributed()) {
87 QuorumPeerMain qp = new QuorumPeerMain();
88 qp.runFromConfig(zkConfig);
89 } else {
90 ZooKeeperServerMain zk = new ZooKeeperServerMain();
91 ServerConfig serverConfig = new ServerConfig();
92 serverConfig.readFrom(zkConfig);
93 zk.runFromConfig(serverConfig);
94 }
95 }
96
97 private static boolean addressIsLocalHost(String address) {
98 return address.equals("localhost") || address.equals("127.0.0.1");
99 }
100
101 private static void writeMyID(Properties properties) throws IOException {
102 long myId = -1;
103
104 Configuration conf = HBaseConfiguration.create();
105 String myAddress = DNS.getDefaultHost(
106 conf.get("hbase.zookeeper.dns.interface","default"),
107 conf.get("hbase.zookeeper.dns.nameserver","default"));
108
109 List<String> ips = new ArrayList<String>();
110
111
112 ips.add(myAddress.contains(".") ?
113 myAddress :
114 StringUtils.simpleHostname(myAddress));
115
116
117 Enumeration<?> nics = NetworkInterface.getNetworkInterfaces();
118 while(nics.hasMoreElements()) {
119 Enumeration<?> rawAdrs =
120 ((NetworkInterface)nics.nextElement()).getInetAddresses();
121 while(rawAdrs.hasMoreElements()) {
122 InetAddress inet = (InetAddress) rawAdrs.nextElement();
123 ips.add(StringUtils.simpleHostname(inet.getHostName()));
124 ips.add(inet.getHostAddress());
125 }
126 }
127
128 for (Entry<Object, Object> entry : properties.entrySet()) {
129 String key = entry.getKey().toString().trim();
130 String value = entry.getValue().toString().trim();
131 if (key.startsWith("server.")) {
132 int dot = key.indexOf('.');
133 long id = Long.parseLong(key.substring(dot + 1));
134 String[] parts = value.split(":");
135 String address = parts[0];
136 if (addressIsLocalHost(address) || ips.contains(address)) {
137 myId = id;
138 break;
139 }
140 }
141 }
142
143 if (myId == -1) {
144 throw new IOException("Could not find my address: " + myAddress +
145 " in list of ZooKeeper quorum servers");
146 }
147
148 String dataDirStr = properties.get("dataDir").toString().trim();
149 File dataDir = new File(dataDirStr);
150 if (!dataDir.isDirectory()) {
151 if (!dataDir.mkdirs()) {
152 throw new IOException("Unable to create data dir " + dataDir);
153 }
154 }
155
156 File myIdFile = new File(dataDir, "myid");
157 PrintWriter w = new PrintWriter(myIdFile);
158 w.println(myId);
159 w.close();
160 }
161
162
163
164
165
166
167
168
169
170 public static Properties makeZKProps(Configuration conf) {
171
172
173 ClassLoader cl = HQuorumPeer.class.getClassLoader();
174 final InputStream inputStream =
175 cl.getResourceAsStream(HConstants.ZOOKEEPER_CONFIG_NAME);
176 if (inputStream != null) {
177 try {
178 return parseZooCfg(conf, inputStream);
179 } catch (IOException e) {
180 LOG.warn("Cannot read " + HConstants.ZOOKEEPER_CONFIG_NAME +
181 ", loading from XML files", e);
182 }
183 }
184
185
186 Properties zkProperties = new Properties();
187
188
189 for (Entry<String, String> entry : conf) {
190 String key = entry.getKey();
191 if (key.startsWith(ZK_CFG_PROPERTY)) {
192 String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE);
193 String value = entry.getValue();
194
195 if (value.contains(VARIABLE_START)) {
196 value = conf.get(key);
197 }
198 zkProperties.put(zkKey, value);
199 }
200 }
201
202
203 if (zkProperties.getProperty(ZK_CLIENT_PORT_KEY) == null) {
204 zkProperties.put(ZK_CLIENT_PORT_KEY,
205 HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
206 }
207
208
209 int peerPort = conf.getInt("hbase.zookeeper.peerport", 2888);
210 int leaderPort = conf.getInt("hbase.zookeeper.leaderport", 3888);
211
212 final String[] serverHosts = conf.getStrings(HConstants.ZOOKEEPER_QUORUM,
213 "localhost");
214 for (int i = 0; i < serverHosts.length; ++i) {
215 String serverHost = serverHosts[i];
216 String address = serverHost + ":" + peerPort + ":" + leaderPort;
217 String key = "server." + i;
218 zkProperties.put(key, address);
219 }
220
221 return zkProperties;
222 }
223
224
225
226
227
228
229
230 public static String getZKQuorumServersString(Properties properties) {
231 String clientPort = null;
232 List<String> servers = new ArrayList<String>();
233
234
235
236 boolean anyValid = false;
237 for (Entry<Object,Object> property : properties.entrySet()) {
238 String key = property.getKey().toString().trim();
239 String value = property.getValue().toString().trim();
240 if (key.equals("clientPort")) {
241 clientPort = value;
242 }
243 else if (key.startsWith("server.")) {
244 String host = value.substring(0, value.indexOf(':'));
245 servers.add(host);
246 try {
247
248 InetAddress.getByName(host);
249 anyValid = true;
250 } catch (UnknownHostException e) {
251 LOG.warn(StringUtils.stringifyException(e));
252 }
253 }
254 }
255
256 if (!anyValid) {
257 LOG.error("no valid quorum servers found in " + HConstants.ZOOKEEPER_CONFIG_NAME);
258 return null;
259 }
260
261 if (clientPort == null) {
262 LOG.error("no clientPort found in " + HConstants.ZOOKEEPER_CONFIG_NAME);
263 return null;
264 }
265
266 if (servers.isEmpty()) {
267 LOG.fatal("No server.X lines found in conf/zoo.cfg. HBase must have a " +
268 "ZooKeeper cluster configured for its operation.");
269 return null;
270 }
271
272 StringBuilder hostPortBuilder = new StringBuilder();
273 for (int i = 0; i < servers.size(); ++i) {
274 String host = servers.get(i);
275 if (i > 0) {
276 hostPortBuilder.append(',');
277 }
278 hostPortBuilder.append(host);
279 hostPortBuilder.append(':');
280 hostPortBuilder.append(clientPort);
281 }
282
283 return hostPortBuilder.toString();
284 }
285
286
287
288
289
290
291
292
293
294 public static Properties parseZooCfg(Configuration conf,
295 InputStream inputStream) throws IOException {
296 Properties properties = new Properties();
297 try {
298 properties.load(inputStream);
299 } catch (IOException e) {
300 final String msg = "fail to read properties from "
301 + HConstants.ZOOKEEPER_CONFIG_NAME;
302 LOG.fatal(msg);
303 throw new IOException(msg, e);
304 }
305 for (Entry<Object, Object> entry : properties.entrySet()) {
306 String value = entry.getValue().toString().trim();
307 String key = entry.getKey().toString().trim();
308 StringBuilder newValue = new StringBuilder();
309 int varStart = value.indexOf(VARIABLE_START);
310 int varEnd = 0;
311 while (varStart != -1) {
312 varEnd = value.indexOf(VARIABLE_END, varStart);
313 if (varEnd == -1) {
314 String msg = "variable at " + varStart + " has no end marker";
315 LOG.fatal(msg);
316 throw new IOException(msg);
317 }
318 String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd);
319
320 String substituteValue = System.getProperty(variable);
321 if (substituteValue == null) {
322 substituteValue = conf.get(variable);
323 }
324 if (substituteValue == null) {
325 String msg = "variable " + variable + " not set in system property "
326 + "or hbase configs";
327 LOG.fatal(msg);
328 throw new IOException(msg);
329 }
330
331 newValue.append(substituteValue);
332
333 varEnd += VARIABLE_END_LENGTH;
334 varStart = value.indexOf(VARIABLE_START, varEnd);
335 }
336
337 if (key.startsWith("server.")) {
338 if (conf.get(HConstants.CLUSTER_DISTRIBUTED).equals(HConstants.CLUSTER_IS_DISTRIBUTED)
339 && value.startsWith("localhost")) {
340 String msg = "The server in zoo.cfg cannot be set to localhost " +
341 "in a fully-distributed setup because it won't be reachable. " +
342 "See \"Getting Started\" for more information.";
343 LOG.fatal(msg);
344 throw new IOException(msg);
345 }
346 }
347 newValue.append(value.substring(varEnd));
348 properties.setProperty(key, newValue.toString());
349 }
350 return properties;
351 }
352 }