View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.mockito.Mockito.mock;
25  import static org.mockito.Mockito.when;
26  
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.HashMap;
30  import java.util.Iterator;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.Map.Entry;
34  import java.util.Queue;
35  import java.util.TreeMap;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.ClusterStatus;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.testclassification.MediumTests;
44  import org.apache.hadoop.hbase.RegionLoad;
45  import org.apache.hadoop.hbase.ServerLoad;
46  import org.apache.hadoop.hbase.ServerName;
47  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
48  import org.apache.hadoop.hbase.master.RackManager;
49  import org.apache.hadoop.hbase.master.RegionPlan;
50  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.net.DNSToSwitchMapping;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  @Category(MediumTests.class)
58  public class TestStochasticLoadBalancer extends BalancerTestBase {
59    public static final String REGION_KEY = "testRegion";
60    private static StochasticLoadBalancer loadBalancer;
61    private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer.class);
62    private static Configuration conf;
63  
64    @BeforeClass
65    public static void beforeAllTests() throws Exception {
66      conf = HBaseConfiguration.create();
67      conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
68      conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
69      conf.setFloat("hbase.regions.slop", 0.0f);
70      loadBalancer = new StochasticLoadBalancer();
71      loadBalancer.setConf(conf);
72    }
73  
74    int[] largeCluster = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
75        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
76        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
77        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
78        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
79        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
80        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
81        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
82        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
83        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
84        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
85        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
86        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
87        0, 0, 0, 0, 0, 56 };
88  
89    // int[testnum][servernumber] -> numregions
90    int[][] clusterStateMocks = new int[][]{
91        // 1 node
92        new int[]{0},
93        new int[]{1},
94        new int[]{10},
95        // 2 node
96        new int[]{0, 0},
97        new int[]{2, 0},
98        new int[]{2, 1},
99        new int[]{2, 2},
100       new int[]{2, 3},
101       new int[]{2, 4},
102       new int[]{1, 1},
103       new int[]{0, 1},
104       new int[]{10, 1},
105       new int[]{514, 1432},
106       new int[]{48, 53},
107       // 3 node
108       new int[]{0, 1, 2},
109       new int[]{1, 2, 3},
110       new int[]{0, 2, 2},
111       new int[]{0, 3, 0},
112       new int[]{0, 4, 0},
113       new int[]{20, 20, 0},
114       // 4 node
115       new int[]{0, 1, 2, 3},
116       new int[]{4, 0, 0, 0},
117       new int[]{5, 0, 0, 0},
118       new int[]{6, 6, 0, 0},
119       new int[]{6, 2, 0, 0},
120       new int[]{6, 1, 0, 0},
121       new int[]{6, 0, 0, 0},
122       new int[]{4, 4, 4, 7},
123       new int[]{4, 4, 4, 8},
124       new int[]{0, 0, 0, 7},
125       // 5 node
126       new int[]{1, 1, 1, 1, 4},
127       // more nodes
128       new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
129       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
130       new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
131       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
132       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
133       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
134       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
135       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
136       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
137       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
138       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
139       new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
140       new int[]{10, 7, 12, 8, 11, 10, 9, 14},
141       new int[]{13, 14, 6, 10, 10, 10, 8, 10},
142       new int[]{130, 14, 60, 10, 100, 10, 80, 10},
143       new int[]{130, 140, 60, 100, 100, 100, 80, 100},
144       largeCluster,
145 
146   };
147 
148   @Test
149   public void testKeepRegionLoad() throws Exception {
150 
151     ServerName sn = ServerName.valueOf("test:8080", 100);
152     int numClusterStatusToAdd = 20000;
153     for (int i = 0; i < numClusterStatusToAdd; i++) {
154       ServerLoad sl = mock(ServerLoad.class);
155 
156       RegionLoad rl = mock(RegionLoad.class);
157       when(rl.getStores()).thenReturn(i);
158 
159       Map<byte[], RegionLoad> regionLoadMap =
160           new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
161       regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl);
162       when(sl.getRegionsLoad()).thenReturn(regionLoadMap);
163 
164       ClusterStatus clusterStatus = mock(ClusterStatus.class);
165       when(clusterStatus.getServers()).thenReturn(Arrays.asList(sn));
166       when(clusterStatus.getLoad(sn)).thenReturn(sl);
167 
168       loadBalancer.setClusterStatus(clusterStatus);
169     }
170     assertTrue(loadBalancer.loads.get(REGION_KEY) != null);
171     assertTrue(loadBalancer.loads.get(REGION_KEY).size() == 15);
172 
173     Queue<RegionLoad> loads = loadBalancer.loads.get(REGION_KEY);
174     int i = 0;
175     while(loads.size() > 0) {
176       RegionLoad rl = loads.remove();
177       assertEquals(i + (numClusterStatusToAdd - 15), rl.getStores());
178       i ++;
179     }
180   }
181 
182   /**
183    * Test the load balancing algorithm.
184    *
185    * Invariant is that all servers should be hosting either floor(average) or
186    * ceiling(average)
187    *
188    * @throws Exception
189    */
190   @Test
191   public void testBalanceCluster() throws Exception {
192 
193     for (int[] mockCluster : clusterStateMocks) {
194       Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
195       List<ServerAndLoad> list = convertToList(servers);
196       LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
197       List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
198       List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers);
199       LOG.info("Mock Balance : " + printMock(balancedCluster));
200       assertClusterAsBalanced(balancedCluster);
201       List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(servers);
202       assertNull(secondPlans);
203       for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
204         returnRegions(entry.getValue());
205         returnServer(entry.getKey());
206       }
207     }
208 
209   }
210 
211   @Test
212   public void testSkewCost() {
213     Configuration conf = HBaseConfiguration.create();
214     StochasticLoadBalancer.CostFunction
215         costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
216     for (int[] mockCluster : clusterStateMocks) {
217       costFunction.init(mockCluster(mockCluster));
218       double cost = costFunction.cost();
219       assertTrue(cost >= 0);
220       assertTrue(cost <= 1.01);
221     }
222 
223     costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
224     assertEquals(0,costFunction.cost(), 0.01);
225     costFunction.init(mockCluster(new int[]{0, 0, 0, 1, 1}));
226     assertEquals(0, costFunction.cost(), 0.01);
227     costFunction.init(mockCluster(new int[]{0, 0, 1, 1, 1}));
228     assertEquals(0, costFunction.cost(), 0.01);
229     costFunction.init(mockCluster(new int[]{0, 1, 1, 1, 1}));
230     assertEquals(0, costFunction.cost(), 0.01);
231     costFunction.init(mockCluster(new int[]{1, 1, 1, 1, 1}));
232     assertEquals(0, costFunction.cost(), 0.01);
233     costFunction.init(mockCluster(new int[]{10000, 0, 0, 0, 0}));
234     assertEquals(1, costFunction.cost(), 0.01);
235   }
236 
237   @Test
238   public void testTableSkewCost() {
239     Configuration conf = HBaseConfiguration.create();
240     StochasticLoadBalancer.CostFunction
241         costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
242     for (int[] mockCluster : clusterStateMocks) {
243       BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
244       costFunction.init(cluster);
245       double cost = costFunction.cost();
246       assertTrue(cost >= 0);
247       assertTrue(cost <= 1.01);
248     }
249   }
250 
251   @Test
252   public void testCostFromArray() {
253     Configuration conf = HBaseConfiguration.create();
254     StochasticLoadBalancer.CostFromRegionLoadFunction
255         costFunction = new StochasticLoadBalancer.MemstoreSizeCostFunction(conf);
256     costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
257 
258     double[] statOne = new double[100];
259     for (int i =0; i < 100; i++) {
260       statOne[i] = 10;
261     }
262     assertEquals(0, costFunction.costFromArray(statOne), 0.01);
263 
264     double[] statTwo= new double[101];
265     for (int i =0; i < 100; i++) {
266       statTwo[i] = 0;
267     }
268     statTwo[100] = 100;
269     assertEquals(1, costFunction.costFromArray(statTwo), 0.01);
270 
271     double[] statThree = new double[200];
272     for (int i =0; i < 100; i++) {
273       statThree[i] = (0);
274       statThree[i+100] = 100;
275     }
276     assertEquals(0.5, costFunction.costFromArray(statThree), 0.01);
277   }
278 
279   @Test(timeout =  60000)
280   public void testLosingRs() throws Exception {
281     int numNodes = 3;
282     int numRegions = 20;
283     int numRegionsPerServer = 3; //all servers except one
284     int replication = 1;
285     int numTables = 2;
286 
287     Map<ServerName, List<HRegionInfo>> serverMap =
288         createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
289     List<ServerAndLoad> list = convertToList(serverMap);
290 
291 
292     List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
293     assertNotNull(plans);
294 
295     // Apply the plan to the mock cluster.
296     List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
297 
298     assertClusterAsBalanced(balancedCluster);
299 
300     ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
301 
302     ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);
303 
304     serverMap.put(deadSn, new ArrayList<HRegionInfo>(0));
305 
306     plans = loadBalancer.balanceCluster(serverMap);
307     assertNull(plans);
308   }
309 
310   @Test
311   public void testReplicaCost() {
312     Configuration conf = HBaseConfiguration.create();
313     StochasticLoadBalancer.CostFunction
314         costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
315     for (int[] mockCluster : clusterStateMocks) {
316       BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
317       costFunction.init(cluster);
318       double cost = costFunction.cost();
319       assertTrue(cost >= 0);
320       assertTrue(cost <= 1.01);
321     }
322   }
323 
324   @Test
325   public void testReplicaCostForReplicas() {
326     Configuration conf = HBaseConfiguration.create();
327     StochasticLoadBalancer.CostFunction
328         costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
329 
330     int [] servers = new int[] {3,3,3,3,3};
331     TreeMap<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(servers);
332 
333     BaseLoadBalancer.Cluster cluster;
334 
335     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
336     costFunction.init(cluster);
337     double costWithoutReplicas = costFunction.cost();
338     assertEquals(0, costWithoutReplicas, 0);
339 
340     // replicate the region from first server to the last server
341     HRegionInfo replica1 = RegionReplicaUtil.getRegionInfoForReplica(
342       clusterState.firstEntry().getValue().get(0),1);
343     clusterState.lastEntry().getValue().add(replica1);
344 
345     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
346     costFunction.init(cluster);
347     double costWith1ReplicaDifferentServer = costFunction.cost();
348 
349     assertEquals(0, costWith1ReplicaDifferentServer, 0);
350 
351     // add a third replica to the last server
352     HRegionInfo replica2 = RegionReplicaUtil.getRegionInfoForReplica(replica1, 2);
353     clusterState.lastEntry().getValue().add(replica2);
354 
355     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
356     costFunction.init(cluster);
357     double costWith1ReplicaSameServer = costFunction.cost();
358 
359     assertTrue(costWith1ReplicaDifferentServer < costWith1ReplicaSameServer);
360 
361     // test with replication = 4 for following:
362 
363     HRegionInfo replica3;
364     Iterator<Entry<ServerName, List<HRegionInfo>>> it;
365     Entry<ServerName, List<HRegionInfo>> entry;
366 
367     clusterState = mockClusterServers(servers);
368     it = clusterState.entrySet().iterator();
369     entry = it.next(); //first server
370     HRegionInfo hri = entry.getValue().get(0);
371     replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
372     replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
373     replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
374     entry.getValue().add(replica1);
375     entry.getValue().add(replica2);
376     it.next().getValue().add(replica3); //2nd server
377 
378     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
379     costFunction.init(cluster);
380     double costWith3ReplicasSameServer = costFunction.cost();
381 
382     clusterState = mockClusterServers(servers);
383     hri = clusterState.firstEntry().getValue().get(0);
384     replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
385     replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
386     replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
387 
388     clusterState.firstEntry().getValue().add(replica1);
389     clusterState.lastEntry().getValue().add(replica2);
390     clusterState.lastEntry().getValue().add(replica3);
391 
392     cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
393     costFunction.init(cluster);
394     double costWith2ReplicasOnTwoServers = costFunction.cost();
395 
396     assertTrue(costWith2ReplicasOnTwoServers < costWith3ReplicasSameServer);
397   }
398 
399   @Test
400   public void testNeedsBalanceForColocatedReplicas() {
401     // check for the case where there are two hosts and with one rack, and where
402     // both the replicas are hosted on the same server
403     List<HRegionInfo> regions = randomRegions(1);
404     ServerName s1 = ServerName.valueOf("host1", 1000, 11111);
405     ServerName s2 = ServerName.valueOf("host11", 1000, 11111);
406     Map<ServerName, List<HRegionInfo>> map = new HashMap<ServerName, List<HRegionInfo>>();
407     map.put(s1, regions);
408     regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
409     // until the step above s1 holds two replicas of a region
410     regions = randomRegions(1);
411     map.put(s2, regions);
412     assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null, null)));
413     // check for the case where there are two hosts on the same rack and there are two racks
414     // and both the replicas are on the same rack
415     map.clear();
416     regions = randomRegions(1);
417     List<HRegionInfo> regionsOnS2 = new ArrayList<HRegionInfo>(1);
418     regionsOnS2.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
419     map.put(s1, regions);
420     map.put(s2, regionsOnS2);
421     // add another server so that the cluster has some host on another rack
422     map.put(ServerName.valueOf("host2", 1000, 11111), randomRegions(1));
423     assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null,
424         new ForTestRackManagerOne())));
425   }
426 
427   @Test (timeout = 60000)
428   public void testSmallCluster() {
429     int numNodes = 10;
430     int numRegions = 1000;
431     int numRegionsPerServer = 40; //all servers except one
432     int replication = 1;
433     int numTables = 10;
434     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
435   }
436 
437   @Test (timeout = 60000)
438   public void testSmallCluster2() {
439     int numNodes = 20;
440     int numRegions = 2000;
441     int numRegionsPerServer = 40; //all servers except one
442     int replication = 1;
443     int numTables = 10;
444     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
445   }
446 
447   @Test (timeout = 60000)
448   public void testSmallCluster3() {
449     int numNodes = 20;
450     int numRegions = 2000;
451     int numRegionsPerServer = 1; // all servers except one
452     int replication = 1;
453     int numTables = 10;
454     /* fails because of max moves */
455     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, false);
456   }
457 
458   @Test (timeout = 800000)
459   public void testMidCluster() {
460     int numNodes = 100;
461     int numRegions = 10000;
462     int numRegionsPerServer = 60; // all servers except one
463     int replication = 1;
464     int numTables = 40;
465     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
466   }
467 
468   @Test (timeout = 800000)
469   public void testMidCluster2() {
470     int numNodes = 200;
471     int numRegions = 100000;
472     int numRegionsPerServer = 40; // all servers except one
473     int replication = 1;
474     int numTables = 400;
475     testWithCluster(numNodes,
476         numRegions,
477         numRegionsPerServer,
478         replication,
479         numTables,
480         false, /* num large num regions means may not always get to best balance with one run */
481         false);
482   }
483 
484 
485   @Test (timeout = 800000)
486   public void testMidCluster3() {
487     int numNodes = 100;
488     int numRegions = 2000;
489     int numRegionsPerServer = 9; // all servers except one
490     int replication = 1;
491     int numTables = 110;
492     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
493     // TODO(eclark): Make sure that the tables are well distributed.
494   }
495 
496   @Test
497   public void testLargeCluster() {
498     int numNodes = 1000;
499     int numRegions = 100000; //100 regions per RS
500     int numRegionsPerServer = 80; //all servers except one
501     int numTables = 100;
502     int replication = 1;
503     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
504   }
505 
506   @Test (timeout = 800000)
507   public void testRegionReplicasOnSmallCluster() {
508     int numNodes = 10;
509     int numRegions = 1000;
510     int replication = 3; // 3 replicas per region
511     int numRegionsPerServer = 80; //all regions are mostly balanced
512     int numTables = 10;
513     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
514   }
515 
516   @Test (timeout = 800000)
517   public void testRegionReplicasOnMidCluster() {
518     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
519     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
520     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
521     loadBalancer.setConf(conf);
522     int numNodes = 200;
523     int numRegions = 40 * 200;
524     int replication = 3; // 3 replicas per region
525     int numRegionsPerServer = 30; //all regions are mostly balanced
526     int numTables = 10;
527     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
528   }
529 
530   @Test (timeout = 800000)
531   public void testRegionReplicasOnLargeCluster() {
532     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
533     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
534     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
535     loadBalancer.setConf(conf);
536     int numNodes = 1000;
537     int numRegions = 20 * numNodes; // 20 * replication regions per RS
538     int numRegionsPerServer = 19; // all servers except one
539     int numTables = 100;
540     int replication = 3;
541     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
542   }
543 
544   @Test (timeout = 800000)
545   public void testRegionReplicasOnMidClusterHighReplication() {
546     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 4000000L);
547     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
548     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
549     loadBalancer.setConf(conf);
550     int numNodes = 80;
551     int numRegions = 6 * numNodes;
552     int replication = 80; // 80 replicas per region, one for each server
553     int numRegionsPerServer = 5;
554     int numTables = 10;
555     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, true);
556   }
557 
558   @Test (timeout = 800000)
559   public void testRegionReplicationOnMidClusterSameHosts() {
560     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
561     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
562     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
563     loadBalancer.setConf(conf);
564     int numHosts = 100;
565     int numRegions = 100 * 100;
566     int replication = 3; // 3 replicas per region
567     int numRegionsPerServer = 5;
568     int numTables = 10;
569     Map<ServerName, List<HRegionInfo>> serverMap =
570         createServerMap(numHosts, numRegions, numRegionsPerServer, replication, numTables);
571     int numNodesPerHost = 4;
572 
573     // create a new map with 4 RS per host.
574     Map<ServerName, List<HRegionInfo>> newServerMap = new TreeMap<ServerName, List<HRegionInfo>>(serverMap);
575     for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverMap.entrySet()) {
576       for (int i=1; i < numNodesPerHost; i++) {
577         ServerName s1 = entry.getKey();
578         ServerName s2 = ServerName.valueOf(s1.getHostname(), s1.getPort() + i, 1); // create an RS for the same host
579         newServerMap.put(s2, new ArrayList<HRegionInfo>());
580       }
581     }
582 
583     testWithCluster(newServerMap, null, true, true);
584   }
585 
586   private static class ForTestRackManager extends RackManager {
587     int numRacks;
588     public ForTestRackManager(int numRacks) {
589       this.numRacks = numRacks;
590     }
591     @Override
592     public String getRack(ServerName server) {
593       return "rack_" + (server.hashCode() % numRacks);
594     }
595   }
596 
597   private static class ForTestRackManagerOne extends RackManager {
598   @Override
599     public String getRack(ServerName server) {
600       return server.getHostname().endsWith("1") ? "rack1" : "rack2";
601     }
602   }
603 
604   @Test (timeout = 800000)
605   public void testRegionReplicationOnMidClusterWithRacks() {
606     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 10000000L);
607     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
608     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
609     loadBalancer.setConf(conf);
610     int numNodes = 30;
611     int numRegions = numNodes * 30;
612     int replication = 3; // 3 replicas per region
613     int numRegionsPerServer = 28;
614     int numTables = 10;
615     int numRacks = 4; // all replicas should be on a different rack
616     Map<ServerName, List<HRegionInfo>> serverMap =
617         createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
618     RackManager rm = new ForTestRackManager(numRacks);
619 
620     testWithCluster(serverMap, rm, false, true);
621   }
622 
623   @Test (timeout = 800000)
624   public void testRegionReplicationOnMidClusterReplicationGreaterThanNumNodes() {
625     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
626     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
627     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
628     loadBalancer.setConf(conf);
629     int numNodes = 40;
630     int numRegions = 6 * 50;
631     int replication = 50; // 50 replicas per region, more than numNodes
632     int numRegionsPerServer = 6;
633     int numTables = 10;
634     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, false);
635   }
636 
637   protected void testWithCluster(int numNodes,
638       int numRegions,
639       int numRegionsPerServer,
640       int replication,
641       int numTables,
642       boolean assertFullyBalanced, boolean assertFullyBalancedForReplicas) {
643     Map<ServerName, List<HRegionInfo>> serverMap =
644         createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
645     testWithCluster(serverMap, null, assertFullyBalanced, assertFullyBalancedForReplicas);
646   }
647 
648 
649   protected void testWithCluster(Map<ServerName, List<HRegionInfo>> serverMap,
650       RackManager rackManager, boolean assertFullyBalanced, boolean assertFullyBalancedForReplicas) {
651     List<ServerAndLoad> list = convertToList(serverMap);
652     LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
653 
654     loadBalancer.setRackManager(rackManager);
655     // Run the balancer.
656     List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
657     assertNotNull(plans);
658 
659     // Check to see that this actually got to a stable place.
660     if (assertFullyBalanced || assertFullyBalancedForReplicas) {
661       // Apply the plan to the mock cluster.
662       List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
663 
664       // Print out the cluster loads to make debugging easier.
665       LOG.info("Mock Balance : " + printMock(balancedCluster));
666 
667       if (assertFullyBalanced) {
668         assertClusterAsBalanced(balancedCluster);
669         List<RegionPlan> secondPlans =  loadBalancer.balanceCluster(serverMap);
670         assertNull(secondPlans);
671       }
672 
673       if (assertFullyBalancedForReplicas) {
674         assertRegionReplicaPlacement(serverMap, rackManager);
675       }
676     }
677   }
678 
679   private Map<ServerName, List<HRegionInfo>> createServerMap(int numNodes,
680                                                              int numRegions,
681                                                              int numRegionsPerServer,
682                                                              int replication,
683                                                              int numTables) {
684     //construct a cluster of numNodes, having  a total of numRegions. Each RS will hold
685     //numRegionsPerServer many regions except for the last one, which will host all the
686     //remaining regions
687     int[] cluster = new int[numNodes];
688     for (int i =0; i < numNodes; i++) {
689       cluster[i] = numRegionsPerServer;
690     }
691     cluster[cluster.length - 1] = numRegions - ((cluster.length - 1) * numRegionsPerServer);
692     Map<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(cluster, numTables);
693     if (replication > 0) {
694       // replicate the regions to the same servers
695       for (List<HRegionInfo> regions : clusterState.values()) {
696         int length = regions.size();
697         for (int i = 0; i < length; i++) {
698           for (int r = 1; r < replication ; r++) {
699             regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(i), r));
700           }
701         }
702       }
703     }
704 
705     return clusterState;
706   }
707 }