1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.util;
18  
19  import java.io.BufferedReader;
20  import java.io.BufferedWriter;
21  import java.io.File;
22  import java.io.FileWriter;
23  import java.io.IOException;
24  import java.io.InputStreamReader;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.HashMap;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Scanner;
31  import java.util.TreeMap;
32  
33  import org.apache.commons.io.FileUtils;
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.RawLocalFileSystem;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.MiniHBaseCluster;
41  import org.apache.hadoop.hbase.client.HTable;
42  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
43  
44  /**
45   * A helper class for process-based mini-cluster tests. Unlike
46   * {@link MiniHBaseCluster}, starts daemons as separate processes, allowing to
47   * do real kill testing.
48   */
49  public class ProcessBasedLocalHBaseCluster {
50  
51    private static final String DEFAULT_WORKDIR =
52        "/tmp/hbase-" + System.getenv("USER");
53  
54    private final String hbaseHome;
55    private final String workDir;
56  
57    private int numRegionServers;
58    private final int zkClientPort;
59    private final int masterPort;
60  
61    private final Configuration conf;
62  
63    private static final int MAX_FILE_SIZE_OVERRIDE = 10 * 1000 * 1000;
64  
65    private static final Log LOG = LogFactory.getLog(
66        ProcessBasedLocalHBaseCluster.class);
67  
68    private List<String> daemonPidFiles =
69        Collections.synchronizedList(new ArrayList<String>());;
70  
71    private boolean shutdownHookInstalled;
72  
73    private String hbaseDaemonScript;
74  
75    /**
76     * Constructor. Modifies the passed configuration.
77     * @param hbaseHome the top directory of the HBase source tree
78     */
79    public ProcessBasedLocalHBaseCluster(Configuration conf, String hbaseHome,
80        int numRegionServers) {
81      this.conf = conf;
82      this.hbaseHome = hbaseHome;
83      this.numRegionServers = numRegionServers;
84      this.workDir = DEFAULT_WORKDIR;
85  
86      hbaseDaemonScript = hbaseHome + "/bin/hbase-daemon.sh";
87      zkClientPort = HBaseTestingUtility.randomFreePort();
88      masterPort = HBaseTestingUtility.randomFreePort();
89  
90      conf.set(HConstants.ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
91      conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
92    }
93  
94    public void start() throws IOException {
95      cleanupOldState();
96  
97      // start ZK
98      LOG.info("Starting ZooKeeper");
99      startZK();
100 
101     HBaseTestingUtility.waitForHostPort(HConstants.LOCALHOST, zkClientPort);
102 
103     startMaster();
104     ZKUtil.waitForBaseZNode(conf);
105 
106     for (int idx = 0; idx < numRegionServers; idx++) {
107       startRegionServer(HBaseTestingUtility.randomFreePort());
108     }
109 
110     LOG.info("Waiting for HBase startup by scanning META");
111     int attemptsLeft = 10;
112     while (attemptsLeft-- > 0) {
113       try {
114         new HTable(conf, HConstants.META_TABLE_NAME);
115       } catch (Exception e) {
116         LOG.info("Waiting for HBase to startup. Retries left: " + attemptsLeft,
117             e);
118         Threads.sleep(1000);
119       }
120     }
121 
122     LOG.info("Process-based HBase Cluster with " + numRegionServers +
123         " region servers up and running... \n\n");
124   }
125 
126   public void startRegionServer(int port) {
127     startServer("regionserver", port);
128   }
129 
130   public void startMaster() {
131     startServer("master", 0);
132   }
133 
134   public void killRegionServer(int port) throws IOException {
135     killServer("regionserver", port);
136   }
137 
138   public void killMaster() throws IOException {
139     killServer("master", 0);
140   }
141 
142   public void startZK() {
143     startServer("zookeeper", 0);
144   }
145 
146   private void executeCommand(String command) {
147     ensureShutdownHookInstalled();
148     executeCommand(command, null);
149   }
150 
151   private void executeCommand(String command, Map<String,
152       String> envOverrides) {
153     LOG.debug("Command : " + command);
154 
155     try {
156       String [] envp = null;
157       if (envOverrides != null) {
158         Map<String, String> map = new HashMap<String, String>(
159             System.getenv());
160         map.putAll(envOverrides);
161         envp = new String[map.size()];
162         int idx = 0;
163         for (Map.Entry<String, String> e: map.entrySet()) {
164           envp[idx++] = e.getKey() + "=" + e.getValue();
165         }
166       }
167 
168       Process p = Runtime.getRuntime().exec(command, envp);
169 
170       BufferedReader stdInput = new BufferedReader(
171           new InputStreamReader(p.getInputStream()));
172       BufferedReader stdError = new BufferedReader(
173           new InputStreamReader(p.getErrorStream()));
174 
175       // read the output from the command
176       String s = null;
177       while ((s = stdInput.readLine()) != null) {
178         System.out.println(s);
179       }
180 
181       // read any errors from the attempted command
182       while ((s = stdError.readLine()) != null) {
183         System.out.println(s);
184       }
185     } catch (IOException e) {
186       LOG.error("Error running: " + command, e);
187     }
188   }
189 
190   private void shutdownAllProcesses() {
191     LOG.info("Killing daemons using pid files");
192     final List<String> pidFiles = new ArrayList<String>(daemonPidFiles);
193     for (String pidFile : pidFiles) {
194       int pid = 0;
195       try {
196         pid = readPidFromFile(pidFile);
197       } catch (IOException ex) {
198         LOG.error("Could not kill process with pid from " + pidFile);
199       }
200 
201       if (pid > 0) {
202         LOG.info("Killing pid " + pid + " (" + pidFile + ")");
203         killProcess(pid);
204       }
205     }
206 
207     LOG.info("Waiting a bit to let processes terminate");
208     Threads.sleep(5000);
209   }
210 
211   private void ensureShutdownHookInstalled() {
212     if (shutdownHookInstalled) {
213       return;
214     }
215 
216     Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
217       @Override
218       public void run() {
219         shutdownAllProcesses();
220       }
221     }));
222 
223     shutdownHookInstalled = true;
224   }
225 
226   private void cleanupOldState() {
227     executeCommand("rm -rf " + workDir);
228   }
229 
230   private void writeStringToFile(String s, String fileName) {
231     try {
232       BufferedWriter out = new BufferedWriter(new FileWriter(fileName));
233       out.write(s);
234       out.close();
235     } catch (IOException e) {
236       LOG.error("Error writing to: " + fileName, e);
237     }
238   }
239 
240   private String serverWorkingDir(String serverName, int port) {
241     String dir;
242     if (serverName.equals("regionserver")) {
243       dir = workDir + "/" + serverName + "-" + port;
244     } else {
245       dir = workDir + "/" + serverName;
246     }
247     return dir;
248   }
249 
250   private int getServerPID(String serverName, int port) throws IOException {
251     String pidFile = pidFilePath(serverName, port);
252     return readPidFromFile(pidFile);
253   }
254 
255   private static int readPidFromFile(String pidFile) throws IOException {
256     Scanner scanner = new Scanner(new File(pidFile));
257     try {
258       return scanner.nextInt();
259     } finally {
260       scanner.close();
261     }
262   }
263 
264   private String pidFilePath(String serverName, int port) {
265     String dir = serverWorkingDir(serverName, port);
266     String user = System.getenv("USER");
267     String pidFile = String.format("%s/hbase-%s-%s.pid",
268                                    dir, user, serverName);
269     return pidFile;
270   }
271 
272   private void killServer(String serverName, int port) throws IOException {
273     int pid = getServerPID(serverName, port);
274     if (pid > 0) {
275       LOG.info("Killing " + serverName + "; pid=" + pid);
276       killProcess(pid);
277     }
278   }
279 
280   private void killProcess(int pid) {
281     String cmd = "kill -s KILL " + pid;
282     executeCommand(cmd);
283   }
284 
285   private void startServer(String serverName, int rsPort) {
286     String conf = generateConfig(rsPort);
287 
288     // create working directory for this region server.
289     String dir = serverWorkingDir(serverName, rsPort);
290     executeCommand("mkdir -p " + dir);
291 
292     writeStringToFile(conf, dir + "/" + "hbase-site.xml");
293 
294     Map<String, String> envOverrides = new HashMap<String, String>();
295     envOverrides.put("HBASE_LOG_DIR", dir);
296     envOverrides.put("HBASE_PID_DIR", dir);
297     try {
298       FileUtils.copyFile(
299           new File(hbaseHome, "conf/log4j.properties"),
300           new File(dir, "log4j.properties"));
301     } catch (IOException ex) {
302       LOG.error("Could not install log4j.properties into " + dir);
303     }
304 
305     executeCommand(hbaseDaemonScript + " --config " + dir +
306                    " start " + serverName, envOverrides);
307     daemonPidFiles.add(pidFilePath(serverName, rsPort));
308   }
309 
310   private final String generateConfig(int rsPort) {
311     StringBuilder sb = new StringBuilder();
312     Map<String, Object> confMap = new TreeMap<String, Object>();
313     confMap.put(HConstants.CLUSTER_DISTRIBUTED, true);
314     if (rsPort > 0) {
315       confMap.put(HConstants.REGIONSERVER_PORT, rsPort);
316       confMap.put(HConstants.REGIONSERVER_INFO_PORT_AUTO, true);
317     }
318 
319     confMap.put(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
320     confMap.put(HConstants.MASTER_PORT, masterPort);
321     confMap.put(HConstants.HREGION_MAX_FILESIZE, MAX_FILE_SIZE_OVERRIDE);
322     confMap.put("fs.file.impl", RawLocalFileSystem.class.getName());
323 
324     sb.append("<configuration>\n");
325     for (Map.Entry<String, Object> entry : confMap.entrySet()) {
326       sb.append("  <property>\n");
327       sb.append("    <name>" + entry.getKey() + "</name>\n");
328       sb.append("    <value>" + entry.getValue() + "</value>\n");
329       sb.append("  </property>\n");
330     }
331     sb.append("</configuration>\n");
332     return sb.toString();
333   }
334 
335   public Configuration getConf() {
336     return conf;
337   }
338 
339 }