1   /**
2    * Copyright 2008 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  import java.io.IOException;
23  import java.util.List;
24  import java.util.ArrayList;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  
29  import org.apache.hadoop.hbase.client.HTable;
30  import org.apache.hadoop.hbase.client.Put;
31  
32  import org.apache.hadoop.hbase.regionserver.HRegionServer;
33  import org.apache.hadoop.hbase.regionserver.HRegion;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.hbase.util.JVMClusterUtil;
36  
37  /**
38   * Test whether region rebalancing works. (HBASE-71)
39   */
40  public class TestRegionRebalancing extends HBaseClusterTestCase {
41    final Log LOG = LogFactory.getLog(this.getClass().getName());
42    HTable table;
43  
44    HTableDescriptor desc;
45  
46    final byte[] FIVE_HUNDRED_KBYTES;
47  
48    final byte [] FAMILY_NAME = Bytes.toBytes("col");
49  
50    /** constructor */
51    public TestRegionRebalancing() {
52      super(1);
53      FIVE_HUNDRED_KBYTES = new byte[500 * 1024];
54      for (int i = 0; i < 500 * 1024; i++) {
55        FIVE_HUNDRED_KBYTES[i] = 'x';
56      }
57  
58      desc = new HTableDescriptor("test");
59      desc.addFamily(new HColumnDescriptor(FAMILY_NAME));
60    }
61  
62    /**
63     * Before the hbase cluster starts up, create some dummy regions.
64     */
65    @Override
66    public void preHBaseClusterSetup() throws IOException {
67      // create a 20-region table by writing directly to disk
68      List<byte []> startKeys = new ArrayList<byte []>();
69      startKeys.add(null);
70      for (int i = 10; i < 29; i++) {
71        startKeys.add(Bytes.toBytes("row_" + i));
72      }
73      startKeys.add(null);
74      LOG.info(startKeys.size() + " start keys generated");
75  
76      List<HRegion> regions = new ArrayList<HRegion>();
77      for (int i = 0; i < 20; i++) {
78        regions.add(createAregion(startKeys.get(i), startKeys.get(i+1)));
79      }
80  
81      // Now create the root and meta regions and insert the data regions
82      // created above into the meta
83  
84      createRootAndMetaRegions();
85      for (HRegion region : regions) {
86        HRegion.addRegionToMETA(meta, region);
87      }
88      closeRootAndMeta();
89    }
90  
91    /**
92     * For HBASE-71. Try a few different configurations of starting and stopping
93     * region servers to see if the assignment or regions is pretty balanced.
94     * @throws IOException
95     */
96    public void testRebalancing() throws IOException {
97      table = new HTable(conf, "test");
98      assertEquals("Test table should have 20 regions",
99        20, table.getStartKeys().length);
100 
101     // verify that the region assignments are balanced to start out
102     assertRegionsAreBalanced();
103 
104     LOG.debug("Adding 2nd region server.");
105     // add a region server - total of 2
106     cluster.startRegionServer();
107     assertRegionsAreBalanced();
108 
109     // add a region server - total of 3
110     LOG.debug("Adding 3rd region server.");
111     cluster.startRegionServer();
112     assertRegionsAreBalanced();
113 
114     // kill a region server - total of 2
115     LOG.debug("Killing the 3rd region server.");
116     cluster.stopRegionServer(2, false);
117     cluster.waitOnRegionServer(2);
118     assertRegionsAreBalanced();
119 
120     // start two more region servers - total of 4
121     LOG.debug("Adding 3rd region server");
122     cluster.startRegionServer();
123     LOG.debug("Adding 4th region server");
124     cluster.startRegionServer();
125     assertRegionsAreBalanced();
126 
127     for (int i = 0; i < 6; i++){
128       LOG.debug("Adding " + (i + 5) + "th region server");
129       cluster.startRegionServer();
130     }
131     assertRegionsAreBalanced();
132   }
133 
134   /** figure out how many regions are currently being served. */
135   private int getRegionCount() {
136     int total = 0;
137     for (HRegionServer server : getOnlineRegionServers()) {
138       total += server.getOnlineRegions().size();
139     }
140     return total;
141   }
142 
143   /**
144    * Determine if regions are balanced. Figure out the total, divide by the
145    * number of online servers, then test if each server is +/- 1 of average
146    * rounded up.
147    */
148   private void assertRegionsAreBalanced() {
149     boolean success = false;
150     float slop = conf.getFloat("hbase.regions.slop", (float)0.1);
151     if (slop <= 0) slop = 1;
152 
153     for (int i = 0; i < 5; i++) {
154       success = true;
155       // make sure all the regions are reassigned before we test balance
156       waitForAllRegionsAssigned();
157 
158       int regionCount = getRegionCount();
159       List<HRegionServer> servers = getOnlineRegionServers();
160       double avg = cluster.getMaster().getAverageLoad();
161       int avgLoadPlusSlop = (int)Math.ceil(avg * (1 + slop));
162       int avgLoadMinusSlop = (int)Math.floor(avg * (1 - slop)) - 1;
163       LOG.debug("There are " + servers.size() + " servers and " + regionCount
164         + " regions. Load Average: " + avg + " low border: " + avgLoadMinusSlop
165         + ", up border: " + avgLoadPlusSlop + "; attempt: " + i);
166 
167       for (HRegionServer server : servers) {
168         int serverLoad = server.getOnlineRegions().size();
169         LOG.debug(server.hashCode() + " Avg: " + avg + " actual: " + serverLoad);
170         if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
171             && serverLoad >= avgLoadMinusSlop)) {
172           LOG.debug(server.hashCode() + " Isn't balanced!!! Avg: " + avg +
173               " actual: " + serverLoad + " slop: " + slop);
174           success = false;
175         }
176       }
177 
178       if (!success) {
179         // one or more servers are not balanced. sleep a little to give it a
180         // chance to catch up. then, go back to the retry loop.
181         try {
182           Thread.sleep(10000);
183         } catch (InterruptedException e) {}
184 
185         continue;
186       }
187 
188       // if we get here, all servers were balanced, so we should just return.
189       return;
190     }
191     // if we get here, we tried 5 times and never got to short circuit out of
192     // the retry loop, so this is a failure.
193     fail("After 5 attempts, region assignments were not balanced.");
194   }
195 
196   private List<HRegionServer> getOnlineRegionServers() {
197     List<HRegionServer> list = new ArrayList<HRegionServer>();
198     for (JVMClusterUtil.RegionServerThread rst : cluster.getRegionServerThreads()) {
199       if (rst.getRegionServer().isOnline()) {
200         list.add(rst.getRegionServer());
201       }
202     }
203     return list;
204   }
205 
206   /**
207    * Wait until all the regions are assigned.
208    */
209   private void waitForAllRegionsAssigned() {
210     while (getRegionCount() < 22) {
211     // while (!cluster.getMaster().allRegionsAssigned()) {
212       LOG.debug("Waiting for there to be 22 regions, but there are " + getRegionCount() + " right now.");
213       try {
214         Thread.sleep(1000);
215       } catch (InterruptedException e) {}
216     }
217   }
218 
219   /**
220    * create a region with the specified start and end key and exactly one row
221    * inside.
222    */
223   private HRegion createAregion(byte [] startKey, byte [] endKey)
224   throws IOException {
225     HRegion region = createNewHRegion(desc, startKey, endKey);
226     byte [] keyToWrite = startKey == null ? Bytes.toBytes("row_000") : startKey;
227     Put put = new Put(keyToWrite);
228     put.add(FAMILY_NAME, null, Bytes.toBytes("test"));
229     region.put(put);
230     region.close();
231     region.getLog().closeAndDelete();
232     return region;
233   }
234 }