1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.util;
18
19 import java.io.BufferedReader;
20 import java.io.BufferedWriter;
21 import java.io.File;
22 import java.io.FileWriter;
23 import java.io.IOException;
24 import java.io.InputStreamReader;
25 import java.util.ArrayList;
26 import java.util.Collections;
27 import java.util.HashMap;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Scanner;
31 import java.util.TreeMap;
32
33 import org.apache.commons.io.FileUtils;
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.RawLocalFileSystem;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.MiniHBaseCluster;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
43
44
45
46
47
48
49 public class ProcessBasedLocalHBaseCluster {
50
51 private static final String DEFAULT_WORKDIR =
52 "/tmp/hbase-" + System.getenv("USER");
53
54 private final String hbaseHome;
55 private final String workDir;
56
57 private int numRegionServers;
58 private final int zkClientPort;
59 private final int masterPort;
60
61 private final Configuration conf;
62
63 private static final int MAX_FILE_SIZE_OVERRIDE = 10 * 1000 * 1000;
64
65 private static final Log LOG = LogFactory.getLog(
66 ProcessBasedLocalHBaseCluster.class);
67
68 private List<String> daemonPidFiles =
69 Collections.synchronizedList(new ArrayList<String>());;
70
71 private boolean shutdownHookInstalled;
72
73 private String hbaseDaemonScript;
74
75
76
77
78
79 public ProcessBasedLocalHBaseCluster(Configuration conf, String hbaseHome,
80 int numRegionServers) {
81 this.conf = conf;
82 this.hbaseHome = hbaseHome;
83 this.numRegionServers = numRegionServers;
84 this.workDir = DEFAULT_WORKDIR;
85
86 hbaseDaemonScript = hbaseHome + "/bin/hbase-daemon.sh";
87 zkClientPort = HBaseTestingUtility.randomFreePort();
88 masterPort = HBaseTestingUtility.randomFreePort();
89
90 conf.set(HConstants.ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
91 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
92 }
93
94 public void start() throws IOException {
95 cleanupOldState();
96
97
98 LOG.info("Starting ZooKeeper");
99 startZK();
100
101 HBaseTestingUtility.waitForHostPort(HConstants.LOCALHOST, zkClientPort);
102
103 startMaster();
104 ZKUtil.waitForBaseZNode(conf);
105
106 for (int idx = 0; idx < numRegionServers; idx++) {
107 startRegionServer(HBaseTestingUtility.randomFreePort());
108 }
109
110 LOG.info("Waiting for HBase startup by scanning META");
111 int attemptsLeft = 10;
112 while (attemptsLeft-- > 0) {
113 try {
114 new HTable(conf, HConstants.META_TABLE_NAME);
115 } catch (Exception e) {
116 LOG.info("Waiting for HBase to startup. Retries left: " + attemptsLeft,
117 e);
118 Threads.sleep(1000);
119 }
120 }
121
122 LOG.info("Process-based HBase Cluster with " + numRegionServers +
123 " region servers up and running... \n\n");
124 }
125
126 public void startRegionServer(int port) {
127 startServer("regionserver", port);
128 }
129
130 public void startMaster() {
131 startServer("master", 0);
132 }
133
134 public void killRegionServer(int port) throws IOException {
135 killServer("regionserver", port);
136 }
137
138 public void killMaster() throws IOException {
139 killServer("master", 0);
140 }
141
142 public void startZK() {
143 startServer("zookeeper", 0);
144 }
145
146 private void executeCommand(String command) {
147 ensureShutdownHookInstalled();
148 executeCommand(command, null);
149 }
150
151 private void executeCommand(String command, Map<String,
152 String> envOverrides) {
153 LOG.debug("Command : " + command);
154
155 try {
156 String [] envp = null;
157 if (envOverrides != null) {
158 Map<String, String> map = new HashMap<String, String>(
159 System.getenv());
160 map.putAll(envOverrides);
161 envp = new String[map.size()];
162 int idx = 0;
163 for (Map.Entry<String, String> e: map.entrySet()) {
164 envp[idx++] = e.getKey() + "=" + e.getValue();
165 }
166 }
167
168 Process p = Runtime.getRuntime().exec(command, envp);
169
170 BufferedReader stdInput = new BufferedReader(
171 new InputStreamReader(p.getInputStream()));
172 BufferedReader stdError = new BufferedReader(
173 new InputStreamReader(p.getErrorStream()));
174
175
176 String s = null;
177 while ((s = stdInput.readLine()) != null) {
178 System.out.println(s);
179 }
180
181
182 while ((s = stdError.readLine()) != null) {
183 System.out.println(s);
184 }
185 } catch (IOException e) {
186 LOG.error("Error running: " + command, e);
187 }
188 }
189
190 private void shutdownAllProcesses() {
191 LOG.info("Killing daemons using pid files");
192 final List<String> pidFiles = new ArrayList<String>(daemonPidFiles);
193 for (String pidFile : pidFiles) {
194 int pid = 0;
195 try {
196 pid = readPidFromFile(pidFile);
197 } catch (IOException ex) {
198 LOG.error("Could not kill process with pid from " + pidFile);
199 }
200
201 if (pid > 0) {
202 LOG.info("Killing pid " + pid + " (" + pidFile + ")");
203 killProcess(pid);
204 }
205 }
206
207 LOG.info("Waiting a bit to let processes terminate");
208 Threads.sleep(5000);
209 }
210
211 private void ensureShutdownHookInstalled() {
212 if (shutdownHookInstalled) {
213 return;
214 }
215
216 Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
217 @Override
218 public void run() {
219 shutdownAllProcesses();
220 }
221 }));
222
223 shutdownHookInstalled = true;
224 }
225
226 private void cleanupOldState() {
227 executeCommand("rm -rf " + workDir);
228 }
229
230 private void writeStringToFile(String s, String fileName) {
231 try {
232 BufferedWriter out = new BufferedWriter(new FileWriter(fileName));
233 out.write(s);
234 out.close();
235 } catch (IOException e) {
236 LOG.error("Error writing to: " + fileName, e);
237 }
238 }
239
240 private String serverWorkingDir(String serverName, int port) {
241 String dir;
242 if (serverName.equals("regionserver")) {
243 dir = workDir + "/" + serverName + "-" + port;
244 } else {
245 dir = workDir + "/" + serverName;
246 }
247 return dir;
248 }
249
250 private int getServerPID(String serverName, int port) throws IOException {
251 String pidFile = pidFilePath(serverName, port);
252 return readPidFromFile(pidFile);
253 }
254
255 private static int readPidFromFile(String pidFile) throws IOException {
256 Scanner scanner = new Scanner(new File(pidFile));
257 try {
258 return scanner.nextInt();
259 } finally {
260 scanner.close();
261 }
262 }
263
264 private String pidFilePath(String serverName, int port) {
265 String dir = serverWorkingDir(serverName, port);
266 String user = System.getenv("USER");
267 String pidFile = String.format("%s/hbase-%s-%s.pid",
268 dir, user, serverName);
269 return pidFile;
270 }
271
272 private void killServer(String serverName, int port) throws IOException {
273 int pid = getServerPID(serverName, port);
274 if (pid > 0) {
275 LOG.info("Killing " + serverName + "; pid=" + pid);
276 killProcess(pid);
277 }
278 }
279
280 private void killProcess(int pid) {
281 String cmd = "kill -s KILL " + pid;
282 executeCommand(cmd);
283 }
284
285 private void startServer(String serverName, int rsPort) {
286 String conf = generateConfig(rsPort);
287
288
289 String dir = serverWorkingDir(serverName, rsPort);
290 executeCommand("mkdir -p " + dir);
291
292 writeStringToFile(conf, dir + "/" + "hbase-site.xml");
293
294 Map<String, String> envOverrides = new HashMap<String, String>();
295 envOverrides.put("HBASE_LOG_DIR", dir);
296 envOverrides.put("HBASE_PID_DIR", dir);
297 try {
298 FileUtils.copyFile(
299 new File(hbaseHome, "conf/log4j.properties"),
300 new File(dir, "log4j.properties"));
301 } catch (IOException ex) {
302 LOG.error("Could not install log4j.properties into " + dir);
303 }
304
305 executeCommand(hbaseDaemonScript + " --config " + dir +
306 " start " + serverName, envOverrides);
307 daemonPidFiles.add(pidFilePath(serverName, rsPort));
308 }
309
310 private final String generateConfig(int rsPort) {
311 StringBuilder sb = new StringBuilder();
312 Map<String, Object> confMap = new TreeMap<String, Object>();
313 confMap.put(HConstants.CLUSTER_DISTRIBUTED, true);
314 if (rsPort > 0) {
315 confMap.put(HConstants.REGIONSERVER_PORT, rsPort);
316 confMap.put(HConstants.REGIONSERVER_INFO_PORT_AUTO, true);
317 }
318
319 confMap.put(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
320 confMap.put(HConstants.MASTER_PORT, masterPort);
321 confMap.put(HConstants.HREGION_MAX_FILESIZE, MAX_FILE_SIZE_OVERRIDE);
322 confMap.put("fs.file.impl", RawLocalFileSystem.class.getName());
323
324 sb.append("<configuration>\n");
325 for (Map.Entry<String, Object> entry : confMap.entrySet()) {
326 sb.append(" <property>\n");
327 sb.append(" <name>" + entry.getKey() + "</name>\n");
328 sb.append(" <value>" + entry.getValue() + "</value>\n");
329 sb.append(" </property>\n");
330 }
331 sb.append("</configuration>\n");
332 return sb.toString();
333 }
334
335 public Configuration getConf() {
336 return conf;
337 }
338
339 }