View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.Set;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.HBaseConfiguration;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.MediumTests;
38  import org.apache.hadoop.hbase.ServerName;
39  import org.apache.hadoop.hbase.master.RegionPlan;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(MediumTests.class)
45  public class TestStochasticLoadBalancer extends BalancerTestBase {
46    private static StochasticLoadBalancer loadBalancer;
47    private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer.class);
48  
49    @BeforeClass
50    public static void beforeAllTests() throws Exception {
51      Configuration conf = HBaseConfiguration.create();
52      conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
53      loadBalancer = new StochasticLoadBalancer();
54      loadBalancer.setConf(conf);
55    }
56  
57    // int[testnum][servernumber] -> numregions
58    int[][] clusterStateMocks = new int[][]{
59        // 1 node
60        new int[]{0},
61        new int[]{1},
62        new int[]{10},
63        // 2 node
64        new int[]{0, 0},
65        new int[]{2, 0},
66        new int[]{2, 1},
67        new int[]{2, 2},
68        new int[]{2, 3},
69        new int[]{2, 4},
70        new int[]{1, 1},
71        new int[]{0, 1},
72        new int[]{10, 1},
73        new int[]{514, 1432},
74        new int[]{47, 53},
75        // 3 node
76        new int[]{0, 1, 2},
77        new int[]{1, 2, 3},
78        new int[]{0, 2, 2},
79        new int[]{0, 3, 0},
80        new int[]{0, 4, 0},
81        new int[]{20, 20, 0},
82        // 4 node
83        new int[]{0, 1, 2, 3},
84        new int[]{4, 0, 0, 0},
85        new int[]{5, 0, 0, 0},
86        new int[]{6, 6, 0, 0},
87        new int[]{6, 2, 0, 0},
88        new int[]{6, 1, 0, 0},
89        new int[]{6, 0, 0, 0},
90        new int[]{4, 4, 4, 7},
91        new int[]{4, 4, 4, 8},
92        new int[]{0, 0, 0, 7},
93        // 5 node
94        new int[]{1, 1, 1, 1, 4},
95        // more nodes
96        new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
97        new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
98        new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
99        new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
100       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
101       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
102       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
103       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
104       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
105       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
106       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
107       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
108       new int[]{10, 7, 12, 8, 11, 10, 9, 14},
109       new int[]{13, 14, 6, 10, 10, 10, 8, 10},
110       new int[]{130, 14, 60, 10, 100, 10, 80, 10},
111       new int[]{130, 140, 60, 100, 100, 100, 80, 100}
112   };
113 
114   /**
115    * Test the load balancing algorithm.
116    *
117    * Invariant is that all servers should be hosting either floor(average) or
118    * ceiling(average)
119    *
120    * @throws Exception
121    */
122   @Test
123   public void testBalanceCluster() throws Exception {
124 
125     for (int[] mockCluster : clusterStateMocks) {
126       Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
127       List<ServerAndLoad> list = convertToList(servers);
128       LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
129       List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
130       List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers);
131       LOG.info("Mock Balance : " + printMock(balancedCluster));
132       assertClusterAsBalanced(balancedCluster);
133       List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(servers);
134       assertNull(secondPlans);
135       for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
136         returnRegions(entry.getValue());
137         returnServer(entry.getKey());
138       }
139     }
140 
141   }
142 
143   @Test
144   public void testSkewCost() {
145     Configuration conf = HBaseConfiguration.create();
146     StochasticLoadBalancer.CostFunction
147         costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
148     for (int[] mockCluster : clusterStateMocks) {
149       double cost = costFunction.cost(mockCluster(mockCluster));
150       assertTrue(cost >= 0);
151       assertTrue(cost <= 1.01);
152     }
153     assertEquals(1,
154         costFunction.cost(mockCluster(new int[]{0, 0, 0, 0, 1})), 0.01);
155     assertEquals(.75,
156         costFunction.cost(mockCluster(new int[]{0, 0, 0, 1, 1})), 0.01);
157     assertEquals(.5,
158         costFunction.cost(mockCluster(new int[]{0, 0, 1, 1, 1})), 0.01);
159     assertEquals(.25,
160         costFunction.cost(mockCluster(new int[]{0, 1, 1, 1, 1})), 0.01);
161     assertEquals(0,
162         costFunction.cost(mockCluster(new int[]{1, 1, 1, 1, 1})), 0.01);
163     assertEquals(0,
164         costFunction.cost(mockCluster(new int[]{10, 10, 10, 10, 10})), 0.01);
165   }
166 
167   @Test
168   public void testTableSkewCost() {
169     Configuration conf = HBaseConfiguration.create();
170     StochasticLoadBalancer.CostFunction
171         costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
172     for (int[] mockCluster : clusterStateMocks) {
173       BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
174       double cost = costFunction.cost(cluster);
175       assertTrue(cost >= 0);
176       assertTrue(cost <= 1.01);
177     }
178   }
179 
180   @Test
181   public void testCostFromArray() {
182     Configuration conf = HBaseConfiguration.create();
183     StochasticLoadBalancer.CostFromRegionLoadFunction
184         costFunction = new StochasticLoadBalancer.MemstoreSizeCostFunction(conf);
185 
186     double[] statOne = new double[100];
187     for (int i =0; i < 100; i++) {
188       statOne[i] = 10;
189     }
190     assertEquals(0, costFunction.costFromArray(statOne), 0.01);
191 
192     double[] statTwo= new double[101];
193     for (int i =0; i < 100; i++) {
194       statTwo[i] = 0;
195     }
196     statTwo[100] = 100;
197     assertEquals(1, costFunction.costFromArray(statTwo), 0.01);
198 
199     double[] statThree = new double[200];
200     for (int i =0; i < 100; i++) {
201       statThree[i] = (0);
202       statThree[i+100] = 100;
203     }
204     assertEquals(0.5, costFunction.costFromArray(statThree), 0.01);
205   }
206 
207   @Test(timeout =  60000)
208   public void testLosingRs() throws Exception {
209     int numNodes = 3;
210     int numRegions = 20;
211     int numRegionsPerServer = 3; //all servers except one
212     int numTables = 2;
213 
214     Map<ServerName, List<HRegionInfo>> serverMap =
215         createServerMap(numNodes, numRegions, numRegionsPerServer, numTables);
216     List<ServerAndLoad> list = convertToList(serverMap);
217 
218 
219     List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
220     assertNotNull(plans);
221 
222     // Apply the plan to the mock cluster.
223     List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
224 
225     assertClusterAsBalanced(balancedCluster);
226 
227     ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
228 
229     ServerName deadSn = new ServerName(sn.getHostname(), sn.getPort(), sn.getStartcode() -100);
230 
231     serverMap.put(deadSn, new ArrayList<HRegionInfo>(0));
232 
233     plans = loadBalancer.balanceCluster(serverMap);
234     assertNull(plans);
235   }
236 
237   @Test (timeout = 60000)
238   public void testSmallCluster() {
239     int numNodes = 10;
240     int numRegions = 1000;
241     int numRegionsPerServer = 40; //all servers except one
242     int numTables = 10;
243     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
244   }
245 
246   @Test (timeout = 60000)
247   public void testSmallCluster2() {
248     int numNodes = 20;
249     int numRegions = 2000;
250     int numRegionsPerServer = 40; //all servers except one
251     int numTables = 10;
252     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
253   }
254 
255   @Test (timeout = 60000)
256   public void testSmallCluster3() {
257     int numNodes = 20;
258     int numRegions = 2000;
259     int numRegionsPerServer = 1; // all servers except one
260     int numTables = 10;
261     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, false /* max moves */);
262   }
263 
264   @Test (timeout = 800000)
265   public void testMidCluster() {
266     int numNodes = 100;
267     int numRegions = 10000;
268     int numRegionsPerServer = 60; // all servers except one
269     int numTables = 40;
270     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
271   }
272 
273   @Test (timeout = 800000)
274   public void testMidCluster2() {
275     int numNodes = 200;
276     int numRegions = 100000;
277     int numRegionsPerServer = 40; // all servers except one
278     int numTables = 400;
279     testWithCluster(numNodes,
280         numRegions,
281         numRegionsPerServer,
282         numTables,
283         false /* num large num regions means may not always get to best balance with one run */);
284   }
285 
286 
287   @Test (timeout = 800000)
288   public void testMidCluster3() {
289     int numNodes = 100;
290     int numRegions = 2000;
291     int numRegionsPerServer = 9; // all servers except one
292     int numTables = 110;
293     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
294     // TODO(eclark): Make sure that the tables are well distributed.
295   }
296 
297   @Test
298   public void testLargeCluster() {
299     int numNodes = 1000;
300     int numRegions = 100000; //100 regions per RS
301     int numRegionsPerServer = 80; //all servers except one
302     int numTables = 100;
303     testWithCluster(numNodes, numRegions, numRegionsPerServer, numTables, true);
304   }
305 
306   protected void testWithCluster(int numNodes,
307                                  int numRegions,
308                                  int numRegionsPerServer,
309                                  int numTables,
310                                  boolean assertFullyBalanced) {
311     Map<ServerName, List<HRegionInfo>> serverMap =
312         createServerMap(numNodes, numRegions, numRegionsPerServer, numTables);
313 
314     List<ServerAndLoad> list = convertToList(serverMap);
315     LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
316 
317     // Run the balancer.
318     List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
319     assertNotNull(plans);
320 
321     // Check to see that this actually got to a stable place.
322     if (assertFullyBalanced) {
323       // Apply the plan to the mock cluster.
324       List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
325 
326       // Print out the cluster loads to make debugging easier.
327       LOG.info("Mock Balance : " + printMock(balancedCluster));
328       assertClusterAsBalanced(balancedCluster);
329       List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(serverMap);
330       assertNull(secondPlans);
331     }
332   }
333 
334   private Map<ServerName, List<HRegionInfo>> createServerMap(int numNodes,
335                                                              int numRegions,
336                                                              int numRegionsPerServer,
337                                                              int numTables) {
338     //construct a cluster of numNodes, having  a total of numRegions. Each RS will hold
339     //numRegionsPerServer many regions except for the last one, which will host all the
340     //remaining regions
341     int[] cluster = new int[numNodes];
342     for (int i =0; i < numNodes; i++) {
343       cluster[i] = numRegionsPerServer;
344     }
345     cluster[cluster.length - 1] = numRegions - ((cluster.length - 1) * numRegionsPerServer);
346     return mockClusterServers(cluster, numTables);
347   }
348 }