1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24
25 import java.util.ArrayList;
26 import java.util.HashMap;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Queue;
31 import java.util.Random;
32 import java.util.Set;
33 import java.util.SortedSet;
34 import java.util.TreeMap;
35 import java.util.TreeSet;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.*;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.junit.BeforeClass;
43 import org.junit.Test;
44 import org.junit.experimental.categories.Category;
45
46
47
48
49
50 @Category(MediumTests.class)
51 public class TestDefaultLoadBalancer {
52 private static final Log LOG = LogFactory.getLog(TestDefaultLoadBalancer.class);
53
54 private static LoadBalancer loadBalancer;
55
56 private static Random rand;
57
58 @BeforeClass
59 public static void beforeAllTests() throws Exception {
60 Configuration conf = HBaseConfiguration.create();
61 conf.set("hbase.regions.slop", "0");
62 loadBalancer = new DefaultLoadBalancer();
63 loadBalancer.setConf(conf);
64 rand = new Random();
65 }
66
67
68 int [][] clusterStateMocks = new int [][] {
69
70 new int [] { 0 },
71 new int [] { 1 },
72 new int [] { 10 },
73
74 new int [] { 0, 0 },
75 new int [] { 2, 0 },
76 new int [] { 2, 1 },
77 new int [] { 2, 2 },
78 new int [] { 2, 3 },
79 new int [] { 2, 4 },
80 new int [] { 1, 1 },
81 new int [] { 0, 1 },
82 new int [] { 10, 1 },
83 new int [] { 14, 1432 },
84 new int [] { 47, 53 },
85
86 new int [] { 0, 1, 2 },
87 new int [] { 1, 2, 3 },
88 new int [] { 0, 2, 2 },
89 new int [] { 0, 3, 0 },
90 new int [] { 0, 4, 0 },
91 new int [] { 20, 20, 0 },
92
93 new int [] { 0, 1, 2, 3 },
94 new int [] { 4, 0, 0, 0 },
95 new int [] { 5, 0, 0, 0 },
96 new int [] { 6, 6, 0, 0 },
97 new int [] { 6, 2, 0, 0 },
98 new int [] { 6, 1, 0, 0 },
99 new int [] { 6, 0, 0, 0 },
100 new int [] { 4, 4, 4, 7 },
101 new int [] { 4, 4, 4, 8 },
102 new int [] { 0, 0, 0, 7 },
103
104 new int [] { 1, 1, 1, 1, 4 },
105
106 new int [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
107 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 },
108 new int [] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 },
109 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 },
110 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 },
111 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 },
112 new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 },
113 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 },
114 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 },
115 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 },
116 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 },
117 new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 },
118 new int [] { 0, 0, 144, 1, 1, 1, 1, 1123, 133, 138, 12, 1444 },
119 new int [] { 0, 0, 144, 1, 0, 4, 1, 1123, 133, 138, 12, 1444 },
120 new int [] { 1538, 1392, 1561, 1557, 1535, 1553, 1385, 1542, 1619 }
121 };
122
123 int [][] regionsAndServersMocks = new int [][] {
124
125 new int [] { 0, 0 },
126 new int [] { 0, 1 },
127 new int [] { 1, 1 },
128 new int [] { 2, 1 },
129 new int [] { 10, 1 },
130 new int [] { 1, 2 },
131 new int [] { 2, 2 },
132 new int [] { 3, 2 },
133 new int [] { 1, 3 },
134 new int [] { 2, 3 },
135 new int [] { 3, 3 },
136 new int [] { 25, 3 },
137 new int [] { 2, 10 },
138 new int [] { 2, 100 },
139 new int [] { 12, 10 },
140 new int [] { 12, 100 },
141 };
142
143
144
145
146
147
148
149
150
151 @Test
152 public void testBalanceCluster() throws Exception {
153
154 for(int [] mockCluster : clusterStateMocks) {
155 Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
156 List <ServerAndLoad> list = convertToList(servers);
157 LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
158 List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
159 List<ServerAndLoad> balancedCluster = reconcile(list, plans);
160 LOG.info("Mock Balance : " + printMock(balancedCluster));
161 assertClusterAsBalanced(balancedCluster);
162 for(Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
163 returnRegions(entry.getValue());
164 returnServer(entry.getKey());
165 }
166 }
167
168 }
169
170
171
172
173
174 public void assertClusterAsBalanced(List<ServerAndLoad> servers) {
175 int numServers = servers.size();
176 int numRegions = 0;
177 int maxRegions = 0;
178 int minRegions = Integer.MAX_VALUE;
179 for(ServerAndLoad server : servers) {
180 int nr = server.getLoad();
181 if(nr > maxRegions) {
182 maxRegions = nr;
183 }
184 if(nr < minRegions) {
185 minRegions = nr;
186 }
187 numRegions += nr;
188 }
189 if(maxRegions - minRegions < 2) {
190
191 return;
192 }
193 int min = numRegions / numServers;
194 int max = numRegions % numServers == 0 ? min : min + 1;
195
196 for(ServerAndLoad server : servers) {
197 assertTrue(server.getLoad() <= max);
198 assertTrue(server.getLoad() >= min);
199 }
200 }
201
202
203
204
205
206
207
208
209 @Test
210 public void testImmediateAssignment() throws Exception {
211 for(int [] mock : regionsAndServersMocks) {
212 LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
213 List<HRegionInfo> regions = randomRegions(mock[0]);
214 List<ServerAndLoad> servers = randomServers(mock[1], 0);
215 List<ServerName> list = getListOfServerNames(servers);
216 Map<HRegionInfo,ServerName> assignments =
217 loadBalancer.immediateAssignment(regions, list);
218 assertImmediateAssignment(regions, list, assignments);
219 returnRegions(regions);
220 returnServers(list);
221 }
222 }
223
224
225
226
227
228
229
230 private void assertImmediateAssignment(List<HRegionInfo> regions,
231 List<ServerName> servers, Map<HRegionInfo, ServerName> assignments) {
232 for(HRegionInfo region : regions) {
233 assertTrue(assignments.containsKey(region));
234 }
235 }
236
237
238
239
240
241
242
243
244
245 @Test
246 public void testBulkAssignment() throws Exception {
247 for(int [] mock : regionsAndServersMocks) {
248 LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
249 List<HRegionInfo> regions = randomRegions(mock[0]);
250 List<ServerAndLoad> servers = randomServers(mock[1], 0);
251 List<ServerName> list = getListOfServerNames(servers);
252 Map<ServerName, List<HRegionInfo>> assignments =
253 loadBalancer.roundRobinAssignment(regions, list);
254 float average = (float)regions.size()/servers.size();
255 int min = (int)Math.floor(average);
256 int max = (int)Math.ceil(average);
257 if(assignments != null && !assignments.isEmpty()) {
258 for(List<HRegionInfo> regionList : assignments.values()) {
259 assertTrue(regionList.size() == min || regionList.size() == max);
260 }
261 }
262 returnRegions(regions);
263 returnServers(list);
264 }
265 }
266
267
268
269
270
271
272 @Test
273 public void testRetainAssignment() throws Exception {
274
275 List<ServerAndLoad> servers = randomServers(10, 10);
276 List<HRegionInfo> regions = randomRegions(100);
277 Map<HRegionInfo, ServerName> existing =
278 new TreeMap<HRegionInfo, ServerName>();
279 for (int i = 0; i < regions.size(); i++) {
280 ServerName sn = servers.get(i % servers.size()).getServerName();
281
282
283 ServerName snWithOldStartCode =
284 new ServerName(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10);
285 existing.put(regions.get(i), snWithOldStartCode);
286 }
287 List<ServerName> listOfServerNames = getListOfServerNames(servers);
288 Map<ServerName, List<HRegionInfo>> assignment =
289 loadBalancer.retainAssignment(existing, listOfServerNames);
290 assertRetainedAssignment(existing, listOfServerNames, assignment);
291
292
293 List<ServerAndLoad> servers2 =
294 new ArrayList<ServerAndLoad>(servers);
295 servers2.add(randomServer(10));
296 servers2.add(randomServer(10));
297 listOfServerNames = getListOfServerNames(servers2);
298 assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
299 assertRetainedAssignment(existing, listOfServerNames, assignment);
300
301
302 List<ServerAndLoad> servers3 =
303 new ArrayList<ServerAndLoad>(servers);
304 servers3.remove(0);
305 servers3.remove(0);
306 listOfServerNames = getListOfServerNames(servers3);
307 assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
308 assertRetainedAssignment(existing, listOfServerNames, assignment);
309 }
310
311 private List<ServerName> getListOfServerNames(final List<ServerAndLoad> sals) {
312 List<ServerName> list = new ArrayList<ServerName>();
313 for (ServerAndLoad e: sals) {
314 list.add(e.getServerName());
315 }
316 return list;
317 }
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332 private void assertRetainedAssignment(
333 Map<HRegionInfo, ServerName> existing, List<ServerName> servers,
334 Map<ServerName, List<HRegionInfo>> assignment) {
335
336 Set<ServerName> onlineServerSet = new TreeSet<ServerName>(servers);
337 Set<HRegionInfo> assignedRegions = new TreeSet<HRegionInfo>();
338 for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
339 assertTrue("Region assigned to server that was not listed as online",
340 onlineServerSet.contains(a.getKey()));
341 for (HRegionInfo r : a.getValue()) assignedRegions.add(r);
342 }
343 assertEquals(existing.size(), assignedRegions.size());
344
345
346 Set<String> onlineHostNames = new TreeSet<String>();
347 for (ServerName s : servers) {
348 onlineHostNames.add(s.getHostname());
349 }
350
351 for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
352 ServerName assignedTo = a.getKey();
353 for (HRegionInfo r : a.getValue()) {
354 ServerName address = existing.get(r);
355 if (address != null && onlineHostNames.contains(address.getHostname())) {
356
357
358
359 assertEquals(address.getHostname(), assignedTo.getHostname());
360 }
361 }
362 }
363 }
364
365 private String printStats(List<ServerAndLoad> servers) {
366 int numServers = servers.size();
367 int totalRegions = 0;
368 for(ServerAndLoad server : servers) {
369 totalRegions += server.getLoad();
370 }
371 float average = (float)totalRegions / numServers;
372 int max = (int)Math.ceil(average);
373 int min = (int)Math.floor(average);
374 return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]";
375 }
376
377 private List<ServerAndLoad> convertToList(final Map<ServerName, List<HRegionInfo>> servers) {
378 List<ServerAndLoad> list =
379 new ArrayList<ServerAndLoad>(servers.size());
380 for (Map.Entry<ServerName, List<HRegionInfo>> e: servers.entrySet()) {
381 list.add(new ServerAndLoad(e.getKey(), e.getValue().size()));
382 }
383 return list;
384 }
385
386 private String printMock(List<ServerAndLoad> balancedCluster) {
387 SortedSet<ServerAndLoad> sorted =
388 new TreeSet<ServerAndLoad>(balancedCluster);
389 ServerAndLoad [] arr =
390 sorted.toArray(new ServerAndLoad[sorted.size()]);
391 StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
392 sb.append("{ ");
393 for(int i = 0; i < arr.length; i++) {
394 if (i != 0) {
395 sb.append(" , ");
396 }
397 sb.append(arr[i].getLoad());
398 }
399 sb.append(" }");
400 return sb.toString();
401 }
402
403
404
405
406
407
408
409
410 private List<ServerAndLoad> reconcile(List<ServerAndLoad> list,
411 List<RegionPlan> plans) {
412 List<ServerAndLoad> result =
413 new ArrayList<ServerAndLoad>(list.size());
414 if (plans == null) return result;
415 Map<ServerName, ServerAndLoad> map =
416 new HashMap<ServerName, ServerAndLoad>(list.size());
417 for (RegionPlan plan : plans) {
418 ServerName source = plan.getSource();
419 updateLoad(map, source, -1);
420 ServerName destination = plan.getDestination();
421 updateLoad(map, destination, +1);
422 }
423 result.clear();
424 result.addAll(map.values());
425 return result;
426 }
427
428 private void updateLoad(Map<ServerName, ServerAndLoad> map,
429 final ServerName sn, final int diff) {
430 ServerAndLoad sal = map.get(sn);
431 if (sal == null) return;
432 sal = new ServerAndLoad(sn, sal.getLoad() + diff);
433 map.put(sn, sal);
434 }
435
436 private Map<ServerName, List<HRegionInfo>> mockClusterServers(
437 int [] mockCluster) {
438 int numServers = mockCluster.length;
439 Map<ServerName, List<HRegionInfo>> servers =
440 new TreeMap<ServerName, List<HRegionInfo>>();
441 for(int i = 0; i < numServers; i++) {
442 int numRegions = mockCluster[i];
443 ServerAndLoad sal = randomServer(0);
444 List<HRegionInfo> regions = randomRegions(numRegions);
445 servers.put(sal.getServerName(), regions);
446 }
447 return servers;
448 }
449
450 private Queue<HRegionInfo> regionQueue = new LinkedList<HRegionInfo>();
451 static int regionId = 0;
452
453 private List<HRegionInfo> randomRegions(int numRegions) {
454 List<HRegionInfo> regions = new ArrayList<HRegionInfo>(numRegions);
455 byte [] start = new byte[16];
456 byte [] end = new byte[16];
457 rand.nextBytes(start);
458 rand.nextBytes(end);
459 for(int i=0;i<numRegions;i++) {
460 if(!regionQueue.isEmpty()) {
461 regions.add(regionQueue.poll());
462 continue;
463 }
464 Bytes.putInt(start, 0, numRegions << 1);
465 Bytes.putInt(end, 0, (numRegions << 1) + 1);
466 HRegionInfo hri = new HRegionInfo(
467 Bytes.toBytes("table" + i), start, end,
468 false, regionId++);
469 regions.add(hri);
470 }
471 return regions;
472 }
473
474 private void returnRegions(List<HRegionInfo> regions) {
475 regionQueue.addAll(regions);
476 }
477
478 private Queue<ServerName> serverQueue = new LinkedList<ServerName>();
479
480 private ServerAndLoad randomServer(final int numRegionsPerServer) {
481 if (!this.serverQueue.isEmpty()) {
482 ServerName sn = this.serverQueue.poll();
483 return new ServerAndLoad(sn, numRegionsPerServer);
484 }
485 String host = "server" + rand.nextInt(100000);
486 int port = rand.nextInt(60000);
487 long startCode = rand.nextLong();
488 ServerName sn = new ServerName(host, port, startCode);
489 return new ServerAndLoad(sn, numRegionsPerServer);
490 }
491
492 private List<ServerAndLoad> randomServers(int numServers, int numRegionsPerServer) {
493 List<ServerAndLoad> servers =
494 new ArrayList<ServerAndLoad>(numServers);
495 for (int i = 0; i < numServers; i++) {
496 servers.add(randomServer(numRegionsPerServer));
497 }
498 return servers;
499 }
500
501 private void returnServer(ServerName server) {
502 serverQueue.add(server);
503 }
504
505 private void returnServers(List<ServerName> servers) {
506 this.serverQueue.addAll(servers);
507 }
508
509 @org.junit.Rule
510 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
511 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
512 }
513