1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.balancer;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.Mockito.mock;
25 import static org.mockito.Mockito.when;
26
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.HashMap;
30 import java.util.Iterator;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Map.Entry;
34 import java.util.Queue;
35 import java.util.TreeMap;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.ClusterStatus;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.testclassification.MediumTests;
44 import org.apache.hadoop.hbase.RegionLoad;
45 import org.apache.hadoop.hbase.ServerLoad;
46 import org.apache.hadoop.hbase.ServerName;
47 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
48 import org.apache.hadoop.hbase.master.RackManager;
49 import org.apache.hadoop.hbase.master.RegionPlan;
50 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.net.DNSToSwitchMapping;
53 import org.junit.BeforeClass;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56
57 @Category(MediumTests.class)
58 public class TestStochasticLoadBalancer extends BalancerTestBase {
59 public static final String REGION_KEY = "testRegion";
60 private static StochasticLoadBalancer loadBalancer;
61 private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer.class);
62 private static Configuration conf;
63
64 @BeforeClass
65 public static void beforeAllTests() throws Exception {
66 conf = HBaseConfiguration.create();
67 conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
68 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
69 conf.setFloat("hbase.regions.slop", 0.0f);
70 loadBalancer = new StochasticLoadBalancer();
71 loadBalancer.setConf(conf);
72 }
73
74 int[] largeCluster = new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
75 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
76 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
77 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
78 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
79 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
80 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
81 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
82 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
83 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
84 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
85 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
86 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
87 0, 0, 0, 0, 0, 56 };
88
89
90 int[][] clusterStateMocks = new int[][]{
91
92 new int[]{0},
93 new int[]{1},
94 new int[]{10},
95
96 new int[]{0, 0},
97 new int[]{2, 0},
98 new int[]{2, 1},
99 new int[]{2, 2},
100 new int[]{2, 3},
101 new int[]{2, 4},
102 new int[]{1, 1},
103 new int[]{0, 1},
104 new int[]{10, 1},
105 new int[]{514, 1432},
106 new int[]{48, 53},
107
108 new int[]{0, 1, 2},
109 new int[]{1, 2, 3},
110 new int[]{0, 2, 2},
111 new int[]{0, 3, 0},
112 new int[]{0, 4, 0},
113 new int[]{20, 20, 0},
114
115 new int[]{0, 1, 2, 3},
116 new int[]{4, 0, 0, 0},
117 new int[]{5, 0, 0, 0},
118 new int[]{6, 6, 0, 0},
119 new int[]{6, 2, 0, 0},
120 new int[]{6, 1, 0, 0},
121 new int[]{6, 0, 0, 0},
122 new int[]{4, 4, 4, 7},
123 new int[]{4, 4, 4, 8},
124 new int[]{0, 0, 0, 7},
125
126 new int[]{1, 1, 1, 1, 4},
127
128 new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
129 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},
130 new int[]{6, 6, 5, 6, 6, 6, 6, 6, 6, 1},
131 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 54},
132 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 55},
133 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 56},
134 new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 16},
135 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 8},
136 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 9},
137 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 10},
138 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 123},
139 new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 155},
140 new int[]{10, 7, 12, 8, 11, 10, 9, 14},
141 new int[]{13, 14, 6, 10, 10, 10, 8, 10},
142 new int[]{130, 14, 60, 10, 100, 10, 80, 10},
143 new int[]{130, 140, 60, 100, 100, 100, 80, 100},
144 largeCluster,
145
146 };
147
148 @Test
149 public void testKeepRegionLoad() throws Exception {
150
151 ServerName sn = ServerName.valueOf("test:8080", 100);
152 int numClusterStatusToAdd = 20000;
153 for (int i = 0; i < numClusterStatusToAdd; i++) {
154 ServerLoad sl = mock(ServerLoad.class);
155
156 RegionLoad rl = mock(RegionLoad.class);
157 when(rl.getStores()).thenReturn(i);
158
159 Map<byte[], RegionLoad> regionLoadMap =
160 new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
161 regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl);
162 when(sl.getRegionsLoad()).thenReturn(regionLoadMap);
163
164 ClusterStatus clusterStatus = mock(ClusterStatus.class);
165 when(clusterStatus.getServers()).thenReturn(Arrays.asList(sn));
166 when(clusterStatus.getLoad(sn)).thenReturn(sl);
167
168 loadBalancer.setClusterStatus(clusterStatus);
169 }
170 assertTrue(loadBalancer.loads.get(REGION_KEY) != null);
171 assertTrue(loadBalancer.loads.get(REGION_KEY).size() == 15);
172
173 Queue<RegionLoad> loads = loadBalancer.loads.get(REGION_KEY);
174 int i = 0;
175 while(loads.size() > 0) {
176 RegionLoad rl = loads.remove();
177 assertEquals(i + (numClusterStatusToAdd - 15), rl.getStores());
178 i ++;
179 }
180 }
181
182
183
184
185
186
187
188
189
190 @Test
191 public void testBalanceCluster() throws Exception {
192
193 for (int[] mockCluster : clusterStateMocks) {
194 Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
195 List<ServerAndLoad> list = convertToList(servers);
196 LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
197 List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
198 List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers);
199 LOG.info("Mock Balance : " + printMock(balancedCluster));
200 assertClusterAsBalanced(balancedCluster);
201 List<RegionPlan> secondPlans = loadBalancer.balanceCluster(servers);
202 assertNull(secondPlans);
203 for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
204 returnRegions(entry.getValue());
205 returnServer(entry.getKey());
206 }
207 }
208
209 }
210
211 @Test
212 public void testSkewCost() {
213 Configuration conf = HBaseConfiguration.create();
214 StochasticLoadBalancer.CostFunction
215 costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
216 for (int[] mockCluster : clusterStateMocks) {
217 costFunction.init(mockCluster(mockCluster));
218 double cost = costFunction.cost();
219 assertTrue(cost >= 0);
220 assertTrue(cost <= 1.01);
221 }
222
223 costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
224 assertEquals(0,costFunction.cost(), 0.01);
225 costFunction.init(mockCluster(new int[]{0, 0, 0, 1, 1}));
226 assertEquals(0, costFunction.cost(), 0.01);
227 costFunction.init(mockCluster(new int[]{0, 0, 1, 1, 1}));
228 assertEquals(0, costFunction.cost(), 0.01);
229 costFunction.init(mockCluster(new int[]{0, 1, 1, 1, 1}));
230 assertEquals(0, costFunction.cost(), 0.01);
231 costFunction.init(mockCluster(new int[]{1, 1, 1, 1, 1}));
232 assertEquals(0, costFunction.cost(), 0.01);
233 costFunction.init(mockCluster(new int[]{10000, 0, 0, 0, 0}));
234 assertEquals(1, costFunction.cost(), 0.01);
235 }
236
237 @Test
238 public void testTableSkewCost() {
239 Configuration conf = HBaseConfiguration.create();
240 StochasticLoadBalancer.CostFunction
241 costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
242 for (int[] mockCluster : clusterStateMocks) {
243 BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
244 costFunction.init(cluster);
245 double cost = costFunction.cost();
246 assertTrue(cost >= 0);
247 assertTrue(cost <= 1.01);
248 }
249 }
250
251 @Test
252 public void testCostFromArray() {
253 Configuration conf = HBaseConfiguration.create();
254 StochasticLoadBalancer.CostFromRegionLoadFunction
255 costFunction = new StochasticLoadBalancer.MemstoreSizeCostFunction(conf);
256 costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
257
258 double[] statOne = new double[100];
259 for (int i =0; i < 100; i++) {
260 statOne[i] = 10;
261 }
262 assertEquals(0, costFunction.costFromArray(statOne), 0.01);
263
264 double[] statTwo= new double[101];
265 for (int i =0; i < 100; i++) {
266 statTwo[i] = 0;
267 }
268 statTwo[100] = 100;
269 assertEquals(1, costFunction.costFromArray(statTwo), 0.01);
270
271 double[] statThree = new double[200];
272 for (int i =0; i < 100; i++) {
273 statThree[i] = (0);
274 statThree[i+100] = 100;
275 }
276 assertEquals(0.5, costFunction.costFromArray(statThree), 0.01);
277 }
278
279 @Test(timeout = 60000)
280 public void testLosingRs() throws Exception {
281 int numNodes = 3;
282 int numRegions = 20;
283 int numRegionsPerServer = 3;
284 int replication = 1;
285 int numTables = 2;
286
287 Map<ServerName, List<HRegionInfo>> serverMap =
288 createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
289 List<ServerAndLoad> list = convertToList(serverMap);
290
291
292 List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
293 assertNotNull(plans);
294
295
296 List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
297
298 assertClusterAsBalanced(balancedCluster);
299
300 ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
301
302 ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);
303
304 serverMap.put(deadSn, new ArrayList<HRegionInfo>(0));
305
306 plans = loadBalancer.balanceCluster(serverMap);
307 assertNull(plans);
308 }
309
310 @Test
311 public void testReplicaCost() {
312 Configuration conf = HBaseConfiguration.create();
313 StochasticLoadBalancer.CostFunction
314 costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
315 for (int[] mockCluster : clusterStateMocks) {
316 BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
317 costFunction.init(cluster);
318 double cost = costFunction.cost();
319 assertTrue(cost >= 0);
320 assertTrue(cost <= 1.01);
321 }
322 }
323
324 @Test
325 public void testReplicaCostForReplicas() {
326 Configuration conf = HBaseConfiguration.create();
327 StochasticLoadBalancer.CostFunction
328 costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
329
330 int [] servers = new int[] {3,3,3,3,3};
331 TreeMap<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(servers);
332
333 BaseLoadBalancer.Cluster cluster;
334
335 cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
336 costFunction.init(cluster);
337 double costWithoutReplicas = costFunction.cost();
338 assertEquals(0, costWithoutReplicas, 0);
339
340
341 HRegionInfo replica1 = RegionReplicaUtil.getRegionInfoForReplica(
342 clusterState.firstEntry().getValue().get(0),1);
343 clusterState.lastEntry().getValue().add(replica1);
344
345 cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
346 costFunction.init(cluster);
347 double costWith1ReplicaDifferentServer = costFunction.cost();
348
349 assertEquals(0, costWith1ReplicaDifferentServer, 0);
350
351
352 HRegionInfo replica2 = RegionReplicaUtil.getRegionInfoForReplica(replica1, 2);
353 clusterState.lastEntry().getValue().add(replica2);
354
355 cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
356 costFunction.init(cluster);
357 double costWith1ReplicaSameServer = costFunction.cost();
358
359 assertTrue(costWith1ReplicaDifferentServer < costWith1ReplicaSameServer);
360
361
362
363 HRegionInfo replica3;
364 Iterator<Entry<ServerName, List<HRegionInfo>>> it;
365 Entry<ServerName, List<HRegionInfo>> entry;
366
367 clusterState = mockClusterServers(servers);
368 it = clusterState.entrySet().iterator();
369 entry = it.next();
370 HRegionInfo hri = entry.getValue().get(0);
371 replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
372 replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
373 replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
374 entry.getValue().add(replica1);
375 entry.getValue().add(replica2);
376 it.next().getValue().add(replica3);
377
378 cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
379 costFunction.init(cluster);
380 double costWith3ReplicasSameServer = costFunction.cost();
381
382 clusterState = mockClusterServers(servers);
383 hri = clusterState.firstEntry().getValue().get(0);
384 replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
385 replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
386 replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
387
388 clusterState.firstEntry().getValue().add(replica1);
389 clusterState.lastEntry().getValue().add(replica2);
390 clusterState.lastEntry().getValue().add(replica3);
391
392 cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
393 costFunction.init(cluster);
394 double costWith2ReplicasOnTwoServers = costFunction.cost();
395
396 assertTrue(costWith2ReplicasOnTwoServers < costWith3ReplicasSameServer);
397 }
398
399 @Test
400 public void testNeedsBalanceForColocatedReplicas() {
401
402
403 List<HRegionInfo> regions = randomRegions(1);
404 ServerName s1 = ServerName.valueOf("host1", 1000, 11111);
405 ServerName s2 = ServerName.valueOf("host11", 1000, 11111);
406 Map<ServerName, List<HRegionInfo>> map = new HashMap<ServerName, List<HRegionInfo>>();
407 map.put(s1, regions);
408 regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
409
410 regions = randomRegions(1);
411 map.put(s2, regions);
412 assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null, null)));
413
414
415 map.clear();
416 regions = randomRegions(1);
417 List<HRegionInfo> regionsOnS2 = new ArrayList<HRegionInfo>(1);
418 regionsOnS2.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
419 map.put(s1, regions);
420 map.put(s2, regionsOnS2);
421
422 map.put(ServerName.valueOf("host2", 1000, 11111), randomRegions(1));
423 assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null,
424 new ForTestRackManagerOne())));
425 }
426
427 @Test (timeout = 60000)
428 public void testSmallCluster() {
429 int numNodes = 10;
430 int numRegions = 1000;
431 int numRegionsPerServer = 40;
432 int replication = 1;
433 int numTables = 10;
434 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
435 }
436
437 @Test (timeout = 60000)
438 public void testSmallCluster2() {
439 int numNodes = 20;
440 int numRegions = 2000;
441 int numRegionsPerServer = 40;
442 int replication = 1;
443 int numTables = 10;
444 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
445 }
446
447 @Test (timeout = 60000)
448 public void testSmallCluster3() {
449 int numNodes = 20;
450 int numRegions = 2000;
451 int numRegionsPerServer = 1;
452 int replication = 1;
453 int numTables = 10;
454
455 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, false);
456 }
457
458 @Test (timeout = 800000)
459 public void testMidCluster() {
460 int numNodes = 100;
461 int numRegions = 10000;
462 int numRegionsPerServer = 60;
463 int replication = 1;
464 int numTables = 40;
465 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
466 }
467
468 @Test (timeout = 800000)
469 public void testMidCluster2() {
470 int numNodes = 200;
471 int numRegions = 100000;
472 int numRegionsPerServer = 40;
473 int replication = 1;
474 int numTables = 400;
475 testWithCluster(numNodes,
476 numRegions,
477 numRegionsPerServer,
478 replication,
479 numTables,
480 false,
481 false);
482 }
483
484
485 @Test (timeout = 800000)
486 public void testMidCluster3() {
487 int numNodes = 100;
488 int numRegions = 2000;
489 int numRegionsPerServer = 9;
490 int replication = 1;
491 int numTables = 110;
492 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
493
494 }
495
496 @Test
497 public void testLargeCluster() {
498 int numNodes = 1000;
499 int numRegions = 100000;
500 int numRegionsPerServer = 80;
501 int numTables = 100;
502 int replication = 1;
503 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
504 }
505
506 @Test (timeout = 800000)
507 public void testRegionReplicasOnSmallCluster() {
508 int numNodes = 10;
509 int numRegions = 1000;
510 int replication = 3;
511 int numRegionsPerServer = 80;
512 int numTables = 10;
513 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
514 }
515
516 @Test (timeout = 800000)
517 public void testRegionReplicasOnMidCluster() {
518 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
519 conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
520 conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000);
521 loadBalancer.setConf(conf);
522 int numNodes = 200;
523 int numRegions = 40 * 200;
524 int replication = 3;
525 int numRegionsPerServer = 30;
526 int numTables = 10;
527 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
528 }
529
530 @Test (timeout = 800000)
531 public void testRegionReplicasOnLargeCluster() {
532 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
533 conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
534 conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000);
535 loadBalancer.setConf(conf);
536 int numNodes = 1000;
537 int numRegions = 20 * numNodes;
538 int numRegionsPerServer = 19;
539 int numTables = 100;
540 int replication = 3;
541 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
542 }
543
544 @Test (timeout = 800000)
545 public void testRegionReplicasOnMidClusterHighReplication() {
546 conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 4000000L);
547 conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000);
548 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
549 loadBalancer.setConf(conf);
550 int numNodes = 80;
551 int numRegions = 6 * numNodes;
552 int replication = 80;
553 int numRegionsPerServer = 5;
554 int numTables = 10;
555 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, true);
556 }
557
558 @Test (timeout = 800000)
559 public void testRegionReplicationOnMidClusterSameHosts() {
560 conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
561 conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000);
562 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
563 loadBalancer.setConf(conf);
564 int numHosts = 100;
565 int numRegions = 100 * 100;
566 int replication = 3;
567 int numRegionsPerServer = 5;
568 int numTables = 10;
569 Map<ServerName, List<HRegionInfo>> serverMap =
570 createServerMap(numHosts, numRegions, numRegionsPerServer, replication, numTables);
571 int numNodesPerHost = 4;
572
573
574 Map<ServerName, List<HRegionInfo>> newServerMap = new TreeMap<ServerName, List<HRegionInfo>>(serverMap);
575 for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverMap.entrySet()) {
576 for (int i=1; i < numNodesPerHost; i++) {
577 ServerName s1 = entry.getKey();
578 ServerName s2 = ServerName.valueOf(s1.getHostname(), s1.getPort() + i, 1);
579 newServerMap.put(s2, new ArrayList<HRegionInfo>());
580 }
581 }
582
583 testWithCluster(newServerMap, null, true, true);
584 }
585
586 private static class ForTestRackManager extends RackManager {
587 int numRacks;
588 public ForTestRackManager(int numRacks) {
589 this.numRacks = numRacks;
590 }
591 @Override
592 public String getRack(ServerName server) {
593 return "rack_" + (server.hashCode() % numRacks);
594 }
595 }
596
597 private static class ForTestRackManagerOne extends RackManager {
598 @Override
599 public String getRack(ServerName server) {
600 return server.getHostname().endsWith("1") ? "rack1" : "rack2";
601 }
602 }
603
604 @Test (timeout = 800000)
605 public void testRegionReplicationOnMidClusterWithRacks() {
606 conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 10000000L);
607 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
608 conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000);
609 loadBalancer.setConf(conf);
610 int numNodes = 30;
611 int numRegions = numNodes * 30;
612 int replication = 3;
613 int numRegionsPerServer = 28;
614 int numTables = 10;
615 int numRacks = 4;
616 Map<ServerName, List<HRegionInfo>> serverMap =
617 createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
618 RackManager rm = new ForTestRackManager(numRacks);
619
620 testWithCluster(serverMap, rm, false, true);
621 }
622
623 @Test (timeout = 800000)
624 public void testRegionReplicationOnMidClusterReplicationGreaterThanNumNodes() {
625 conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
626 conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000);
627 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
628 loadBalancer.setConf(conf);
629 int numNodes = 40;
630 int numRegions = 6 * 50;
631 int replication = 50;
632 int numRegionsPerServer = 6;
633 int numTables = 10;
634 testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, false);
635 }
636
637 protected void testWithCluster(int numNodes,
638 int numRegions,
639 int numRegionsPerServer,
640 int replication,
641 int numTables,
642 boolean assertFullyBalanced, boolean assertFullyBalancedForReplicas) {
643 Map<ServerName, List<HRegionInfo>> serverMap =
644 createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
645 testWithCluster(serverMap, null, assertFullyBalanced, assertFullyBalancedForReplicas);
646 }
647
648
649 protected void testWithCluster(Map<ServerName, List<HRegionInfo>> serverMap,
650 RackManager rackManager, boolean assertFullyBalanced, boolean assertFullyBalancedForReplicas) {
651 List<ServerAndLoad> list = convertToList(serverMap);
652 LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
653
654 loadBalancer.setRackManager(rackManager);
655
656 List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
657 assertNotNull(plans);
658
659
660 if (assertFullyBalanced || assertFullyBalancedForReplicas) {
661
662 List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
663
664
665 LOG.info("Mock Balance : " + printMock(balancedCluster));
666
667 if (assertFullyBalanced) {
668 assertClusterAsBalanced(balancedCluster);
669 List<RegionPlan> secondPlans = loadBalancer.balanceCluster(serverMap);
670 assertNull(secondPlans);
671 }
672
673 if (assertFullyBalancedForReplicas) {
674 assertRegionReplicaPlacement(serverMap, rackManager);
675 }
676 }
677 }
678
679 private Map<ServerName, List<HRegionInfo>> createServerMap(int numNodes,
680 int numRegions,
681 int numRegionsPerServer,
682 int replication,
683 int numTables) {
684
685
686
687 int[] cluster = new int[numNodes];
688 for (int i =0; i < numNodes; i++) {
689 cluster[i] = numRegionsPerServer;
690 }
691 cluster[cluster.length - 1] = numRegions - ((cluster.length - 1) * numRegionsPerServer);
692 Map<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(cluster, numTables);
693 if (replication > 0) {
694
695 for (List<HRegionInfo> regions : clusterState.values()) {
696 int length = regions.size();
697 for (int i = 0; i < length; i++) {
698 for (int r = 1; r < replication ; r++) {
699 regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(i), r));
700 }
701 }
702 }
703 }
704
705 return clusterState;
706 }
707 }