1   /**
2    * Copyright 2008 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.Map;
26  import java.util.concurrent.ConcurrentHashMap;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.hbase.client.HConnectionManager;
33  import org.apache.hadoop.hbase.master.HMaster;
34  import org.apache.hadoop.hbase.regionserver.HRegion;
35  import org.apache.hadoop.hbase.regionserver.HRegionServer;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.hbase.util.JVMClusterUtil;
38  import org.apache.hadoop.hbase.util.Threads;
39  import org.apache.hadoop.hdfs.DistributedFileSystem;
40  import org.apache.hadoop.io.MapWritable;
41  import org.apache.hadoop.security.UnixUserGroupInformation;
42  import org.apache.hadoop.security.UserGroupInformation;
43  
44  /**
45   * This class creates a single process HBase cluster.
46   * each server.  The master uses the 'default' FileSystem.  The RegionServers,
47   * if we are running on DistributedFilesystem, create a FileSystem instance
48   * each and will close down their instance on the way out.
49   */
50  public class MiniHBaseCluster {
51    static final Log LOG = LogFactory.getLog(MiniHBaseCluster.class.getName());
52    private Configuration conf;
53    public LocalHBaseCluster hbaseCluster;
54    // Cache this.  For some reason only works first time I get it.  TODO: Figure
55    // out why.
56    private final static UserGroupInformation UGI;
57    static {
58      UGI = UserGroupInformation.getCurrentUGI();
59    }
60  
61    /**
62     * Start a MiniHBaseCluster.
63     * @param conf Configuration to be used for cluster
64     * @param numRegionServers initial number of region servers to start.
65     * @throws IOException
66     */
67    public MiniHBaseCluster(Configuration conf, int numRegionServers)
68    throws IOException {
69      this.conf = conf;
70      conf.set(HConstants.MASTER_PORT, "0");
71      init(numRegionServers);
72    }
73  
74    /**
75     * Override Master so can add inject behaviors testing.
76     */
77    public static class MiniHBaseClusterMaster extends HMaster {
78      private final Map<HServerInfo, List<HMsg>> messages =
79        new ConcurrentHashMap<HServerInfo, List<HMsg>>();
80  
81      private final Map<HServerInfo, IOException> exceptions =
82        new ConcurrentHashMap<HServerInfo, IOException>();
83  
84      public MiniHBaseClusterMaster(final Configuration conf)
85      throws IOException {
86        super(conf);
87      }
88  
89      /**
90       * Add a message to send to a regionserver next time it checks in.
91       * @param hsi RegionServer's HServerInfo.
92       * @param msg Message to add.
93       */
94      void addMessage(final HServerInfo hsi, HMsg msg) {
95        synchronized(this.messages) {
96          List<HMsg> hmsgs = this.messages.get(hsi);
97          if (hmsgs == null) {
98            hmsgs = new ArrayList<HMsg>();
99            this.messages.put(hsi, hmsgs);
100         }
101         hmsgs.add(msg);
102       }
103     }
104 
105     void addException(final HServerInfo hsi, final IOException ex) {
106       this.exceptions.put(hsi, ex);
107     }
108 
109     /**
110      * This implementation is special, exceptions will be treated first and
111      * message won't be sent back to the region servers even if some are
112      * specified.
113      * @param hsi the rs
114      * @param msgs Messages to add to
115      * @return
116      * @throws IOException will be throw if any added for this region server
117      */
118     @Override
119     protected HMsg[] adornRegionServerAnswer(final HServerInfo hsi,
120         final HMsg[] msgs) throws IOException {
121       IOException ex = this.exceptions.remove(hsi);
122       if (ex != null) {
123         throw ex;
124       }
125       HMsg [] answerMsgs = msgs;
126       synchronized (this.messages) {
127         List<HMsg> hmsgs = this.messages.get(hsi);
128         if (hmsgs != null && !hmsgs.isEmpty()) {
129           int size = answerMsgs.length;
130           HMsg [] newAnswerMsgs = new HMsg[size + hmsgs.size()];
131           System.arraycopy(answerMsgs, 0, newAnswerMsgs, 0, answerMsgs.length);
132           for (int i = 0; i < hmsgs.size(); i++) {
133             newAnswerMsgs[answerMsgs.length + i] = hmsgs.get(i);
134           }
135           answerMsgs = newAnswerMsgs;
136           hmsgs.clear();
137         }
138       }
139       return super.adornRegionServerAnswer(hsi, answerMsgs);
140     }
141   }
142 
143   /**
144    * Subclass so can get at protected methods (none at moment).  Also, creates
145    * a FileSystem instance per instantiation.  Adds a shutdown own FileSystem
146    * on the way out. Shuts down own Filesystem only, not All filesystems as
147    * the FileSystem system exit hook does.
148    */
149   public static class MiniHBaseClusterRegionServer extends HRegionServer {
150     private static int index = 0;
151     private Thread shutdownThread = null;
152 
153     public MiniHBaseClusterRegionServer(Configuration conf)
154         throws IOException {
155       super(setDifferentUser(conf));
156     }
157 
158     public void setHServerInfo(final HServerInfo hsi) {
159       this.serverInfo = hsi;
160     }
161 
162     /*
163      * @param c
164      * @param currentfs We return this if we did not make a new one.
165      * @param uniqueName Same name used to help identify the created fs.
166      * @return A new fs instance if we are up on DistributeFileSystem.
167      * @throws IOException
168      */
169     private static Configuration setDifferentUser(final Configuration c)
170     throws IOException {
171       FileSystem currentfs = FileSystem.get(c);
172       if (!(currentfs instanceof DistributedFileSystem)) return c;
173       // Else distributed filesystem.  Make a new instance per daemon.  Below
174       // code is taken from the AppendTestUtil over in hdfs.
175       Configuration c2 = new Configuration(c);
176       String username = UGI.getUserName() + ".hrs." + index++;
177       UnixUserGroupInformation.saveToConf(c2,
178         UnixUserGroupInformation.UGI_PROPERTY_NAME,
179         new UnixUserGroupInformation(username, new String[]{"supergroup"}));
180       return c2;
181     }
182 
183     @Override
184     protected void init(MapWritable c) throws IOException {
185       super.init(c);
186       // Run this thread to shutdown our filesystem on way out.
187       this.shutdownThread = new SingleFileSystemShutdownThread(getFileSystem());
188     }
189 
190     @Override
191     public void run() {
192       try {
193         super.run();
194       } finally {
195         // Run this on the way out.
196         if (this.shutdownThread != null) {
197           this.shutdownThread.start();
198           Threads.shutdown(this.shutdownThread, 30000);
199         }
200       }
201     }
202 
203     public void kill() {
204       super.kill();
205     }
206   }
207 
208   /**
209    * Alternate shutdown hook.
210    * Just shuts down the passed fs, not all as default filesystem hook does.
211    */
212   static class SingleFileSystemShutdownThread extends Thread {
213     private final FileSystem fs;
214     SingleFileSystemShutdownThread(final FileSystem fs) {
215       super("Shutdown of " + fs);
216       this.fs = fs;
217     }
218     @Override
219     public void run() {
220       try {
221         LOG.info("Hook closing fs=" + this.fs);
222         this.fs.close();
223       } catch (IOException e) {
224         LOG.warn("Running hook", e);
225       }
226     }
227   }
228 
229   private void init(final int nRegionNodes) throws IOException {
230     try {
231       // start up a LocalHBaseCluster
232       hbaseCluster = new LocalHBaseCluster(conf, nRegionNodes,
233           MiniHBaseCluster.MiniHBaseClusterMaster.class,
234           MiniHBaseCluster.MiniHBaseClusterRegionServer.class);
235       hbaseCluster.startup();
236     } catch(IOException e) {
237       shutdown();
238       throw e;
239     }
240   }
241 
242   /**
243    * Starts a region server thread running
244    *
245    * @throws IOException
246    * @return New RegionServerThread
247    */
248   public JVMClusterUtil.RegionServerThread startRegionServer() throws IOException {
249     JVMClusterUtil.RegionServerThread t = this.hbaseCluster.addRegionServer();
250     t.start();
251     t.waitForServerOnline();
252     return t;
253   }
254 
255   /**
256    * @return Returns the rpc address actually used by the master server, because
257    * the supplied port is not necessarily the actual port used.
258    */
259   public HServerAddress getHMasterAddress() {
260     return this.hbaseCluster.getMaster().getMasterAddress();
261   }
262 
263   /**
264    * @return the HMaster
265    */
266   public HMaster getMaster() {
267     return this.hbaseCluster.getMaster();
268   }
269 
270   /**
271    * Cause a region server to exit doing basic clean up only on its way out.
272    * @param serverNumber  Used as index into a list.
273    */
274   public String abortRegionServer(int serverNumber) {
275     HRegionServer server = getRegionServer(serverNumber);
276     LOG.info("Aborting " + server.toString());
277     server.abort("Aborting for tests", new Exception("Trace info"));
278     return server.toString();
279   }
280 
281   /**
282    * Shut down the specified region server cleanly
283    *
284    * @param serverNumber  Used as index into a list.
285    * @return the region server that was stopped
286    */
287   public JVMClusterUtil.RegionServerThread stopRegionServer(int serverNumber) {
288     return stopRegionServer(serverNumber, true);
289   }
290 
291   /**
292    * Shut down the specified region server cleanly
293    *
294    * @param serverNumber  Used as index into a list.
295    * @param shutdownFS True is we are to shutdown the filesystem as part of this
296    * regionserver's shutdown.  Usually we do but you do not want to do this if
297    * you are running multiple regionservers in a test and you shut down one
298    * before end of the test.
299    * @return the region server that was stopped
300    */
301   public JVMClusterUtil.RegionServerThread stopRegionServer(int serverNumber,
302       final boolean shutdownFS) {
303     JVMClusterUtil.RegionServerThread server =
304       hbaseCluster.getRegionServers().get(serverNumber);
305     LOG.info("Stopping " + server.toString());
306     server.getRegionServer().stop();
307     return server;
308   }
309 
310   /**
311    * Wait for the specified region server to stop. Removes this thread from list
312    * of running threads.
313    * @param serverNumber
314    * @return Name of region server that just went down.
315    */
316   public String waitOnRegionServer(final int serverNumber) {
317     return this.hbaseCluster.waitOnRegionServer(serverNumber);
318   }
319 
320   /**
321    * Wait for Mini HBase Cluster to shut down.
322    */
323   public void join() {
324     this.hbaseCluster.join();
325   }
326 
327   /**
328    * Shut down the mini HBase cluster
329    * @throws IOException
330    */
331   public void shutdown() throws IOException {
332     if (this.hbaseCluster != null) {
333       this.hbaseCluster.shutdown();
334     }
335     HConnectionManager.deleteAllConnections(false);
336   }
337 
338   /**
339    * Call flushCache on all regions on all participating regionservers.
340    * @throws IOException
341    */
342   public void flushcache() throws IOException {
343     for (JVMClusterUtil.RegionServerThread t:
344         this.hbaseCluster.getRegionServers()) {
345       for(HRegion r: t.getRegionServer().getOnlineRegions()) {
346         r.flushcache();
347       }
348     }
349   }
350 
351   /**
352    * Call flushCache on all regions of the specified table.
353    * @throws IOException
354    */
355   public void flushcache(byte [] tableName) throws IOException {
356     for (JVMClusterUtil.RegionServerThread t:
357         this.hbaseCluster.getRegionServers()) {
358       for(HRegion r: t.getRegionServer().getOnlineRegions()) {
359         if(Bytes.equals(r.getTableDesc().getName(), tableName)) {
360           r.flushcache();
361         }
362       }
363     }
364   }
365 
366   /**
367    * @return List of region server threads.
368    */
369   public List<JVMClusterUtil.RegionServerThread> getRegionServerThreads() {
370     return this.hbaseCluster.getRegionServers();
371   }
372 
373   /**
374    * @return List of live region server threads (skips the aborted and the killed)
375    */
376   public List<JVMClusterUtil.RegionServerThread> getLiveRegionServerThreads() {
377     return this.hbaseCluster.getLiveRegionServers();
378   }
379 
380   /**
381    * Grab a numbered region server of your choice.
382    * @param serverNumber
383    * @return region server
384    */
385   public HRegionServer getRegionServer(int serverNumber) {
386     return hbaseCluster.getRegionServer(serverNumber);
387   }
388 
389   public List<HRegion> getRegions(byte[] tableName) {
390     List<HRegion> ret = new ArrayList<HRegion>();
391     for (JVMClusterUtil.RegionServerThread rst : getRegionServerThreads()) {
392       HRegionServer hrs = rst.getRegionServer();
393       for (HRegion region : hrs.getOnlineRegions()) {
394         if (Bytes.equals(region.getTableDesc().getName(), tableName)) {
395           ret.add(region);
396         }
397       }
398     }
399     return ret;
400   }
401 
402   /**
403    * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()}
404    * of HRS carrying regionName. Returns -1 if none found.
405    */
406   public int getServerWithMeta() {
407     return getServerWith(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
408   }
409 
410   /**
411    * Get the location of the specified region
412    * @param regionName Name of the region in bytes
413    * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()}
414    * of HRS carrying .META.. Returns -1 if none found.
415    */
416   public int getServerWith(byte[] regionName) {
417     int index = -1;
418     int count = 0;
419     for (JVMClusterUtil.RegionServerThread rst: getRegionServerThreads()) {
420       HRegionServer hrs = rst.getRegionServer();
421       HRegion metaRegion =
422         hrs.getOnlineRegion(regionName);
423       if (metaRegion != null) {
424         index = count;
425         break;
426       }
427       count++;
428     }
429     return index;
430   }
431 
432   /**
433    * Add an exception to send when a region server checks back in
434    * @param serverNumber Which server to send it to
435    * @param ex The exception that will be sent
436    * @throws IOException
437    */
438   public void addExceptionToSendRegionServer(final int serverNumber,
439       IOException ex) throws IOException {
440     MiniHBaseClusterRegionServer hrs =
441       (MiniHBaseClusterRegionServer)getRegionServer(serverNumber);
442     addExceptionToSendRegionServer(hrs, ex);
443   }
444 
445   /**
446    * Add an exception to send when a region server checks back in
447    * @param hrs Which server to send it to
448    * @param ex The exception that will be sent
449    * @throws IOException
450    */
451   public void addExceptionToSendRegionServer(
452       final MiniHBaseClusterRegionServer hrs, IOException ex)
453       throws IOException {
454     ((MiniHBaseClusterMaster)getMaster()).addException(hrs.getHServerInfo(),ex);
455   }
456 
457   /**
458    * Add a message to include in the responses send a regionserver when it
459    * checks back in.
460    * @param serverNumber Which server to send it to.
461    * @param msg The MESSAGE
462    * @throws IOException
463    */
464   public void addMessageToSendRegionServer(final int serverNumber,
465     final HMsg msg)
466   throws IOException {
467     MiniHBaseClusterRegionServer hrs =
468       (MiniHBaseClusterRegionServer)getRegionServer(serverNumber);
469     addMessageToSendRegionServer(hrs, msg);
470   }
471 
472   /**
473    * Add a message to include in the responses send a regionserver when it
474    * checks back in.
475    * @param hrs Which region server.
476    * @param msg The MESSAGE
477    * @throws IOException
478    */
479   public void addMessageToSendRegionServer(final MiniHBaseClusterRegionServer hrs,
480     final HMsg msg)
481   throws IOException {
482     ((MiniHBaseClusterMaster)getMaster()).addMessage(hrs.getHServerInfo(), msg);
483   }
484 }