1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.LinkedList;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Queue;
31  import java.util.Random;
32  import java.util.Set;
33  import java.util.SortedSet;
34  import java.util.TreeMap;
35  import java.util.TreeSet;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.*;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.junit.BeforeClass;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  
46  
47  /**
48   * Test the load balancer that is created by default.
49   */
50  @Category(MediumTests.class)
51  public class TestDefaultLoadBalancer {
52    private static final Log LOG = LogFactory.getLog(TestDefaultLoadBalancer.class);
53  
54    private static LoadBalancer loadBalancer;
55  
56    private static Random rand;
57  
58    @BeforeClass
59    public static void beforeAllTests() throws Exception {
60      Configuration conf = HBaseConfiguration.create();
61      conf.set("hbase.regions.slop", "0");
62      loadBalancer = new DefaultLoadBalancer();
63      loadBalancer.setConf(conf);
64      rand = new Random();
65    }
66  
67    // int[testnum][servernumber] -> numregions
68    int [][] clusterStateMocks = new int [][] {
69        // 1 node
70        new int [] { 0 },
71        new int [] { 1 },
72        new int [] { 10 },
73        // 2 node
74        new int [] { 0, 0 },
75        new int [] { 2, 0 },
76        new int [] { 2, 1 },
77        new int [] { 2, 2 },
78        new int [] { 2, 3 },
79        new int [] { 2, 4 },
80        new int [] { 1, 1 },
81        new int [] { 0, 1 },
82        new int [] { 10, 1 },
83        new int [] { 14, 1432 },
84        new int [] { 47, 53 },
85        // 3 node
86        new int [] { 0, 1, 2 },
87        new int [] { 1, 2, 3 },
88        new int [] { 0, 2, 2 },
89        new int [] { 0, 3, 0 },
90        new int [] { 0, 4, 0 },
91        new int [] { 20, 20, 0 },
92        // 4 node
93        new int [] { 0, 1, 2, 3 },
94        new int [] { 4, 0, 0, 0 },
95        new int [] { 5, 0, 0, 0 },
96        new int [] { 6, 6, 0, 0 },
97        new int [] { 6, 2, 0, 0 },
98        new int [] { 6, 1, 0, 0 },
99        new int [] { 6, 0, 0, 0 },
100       new int [] { 4, 4, 4, 7 },
101       new int [] { 4, 4, 4, 8 },
102       new int [] { 0, 0, 0, 7 },
103       // 5 node
104       new int [] { 1, 1, 1, 1, 4 },
105       // more nodes
106       new int [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
107       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 },
108       new int [] { 6, 6, 5, 6, 6, 6, 6, 6, 6, 1 },
109       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 54 },
110       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 55 },
111       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 },
112       new int [] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 },
113       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 8 },
114       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 },
115       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 10 },
116       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 123 },
117       new int [] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 155 },
118       new int [] { 0, 0, 144, 1, 1, 1, 1, 1123, 133, 138, 12, 1444 },
119       new int [] { 0, 0, 144, 1, 0, 4, 1, 1123, 133, 138, 12, 1444 },
120       new int [] { 1538, 1392, 1561, 1557, 1535, 1553, 1385, 1542, 1619 }
121   };
122 
123   int [][] regionsAndServersMocks = new int [][] {
124       // { num regions, num servers }
125       new int [] { 0, 0 },
126       new int [] { 0, 1 },
127       new int [] { 1, 1 },
128       new int [] { 2, 1 },
129       new int [] { 10, 1 },
130       new int [] { 1, 2 },
131       new int [] { 2, 2 },
132       new int [] { 3, 2 },
133       new int [] { 1, 3 },
134       new int [] { 2, 3 },
135       new int [] { 3, 3 },
136       new int [] { 25, 3 },
137       new int [] { 2, 10 },
138       new int [] { 2, 100 },
139       new int [] { 12, 10 },
140       new int [] { 12, 100 },
141   };
142 
143   /**
144    * Test the load balancing algorithm.
145    *
146    * Invariant is that all servers should be hosting either
147    * floor(average) or ceiling(average)
148    *
149    * @throws Exception
150    */
151   @Test
152   public void testBalanceCluster() throws Exception {
153 
154     for(int [] mockCluster : clusterStateMocks) {
155       Map<ServerName, List<HRegionInfo>> servers =  mockClusterServers(mockCluster);
156       List <ServerAndLoad> list = convertToList(servers);
157       LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
158       List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
159       List<ServerAndLoad> balancedCluster = reconcile(list, plans);
160       LOG.info("Mock Balance : " + printMock(balancedCluster));
161       assertClusterAsBalanced(balancedCluster);
162       for(Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
163         returnRegions(entry.getValue());
164         returnServer(entry.getKey());
165       }
166     }
167 
168   }
169 
170   /**
171    * Invariant is that all servers have between floor(avg) and ceiling(avg)
172    * number of regions.
173    */
174   public void assertClusterAsBalanced(List<ServerAndLoad> servers) {
175     int numServers = servers.size();
176     int numRegions = 0;
177     int maxRegions = 0;
178     int minRegions = Integer.MAX_VALUE;
179     for(ServerAndLoad server : servers) {
180       int nr = server.getLoad();
181       if(nr > maxRegions) {
182         maxRegions = nr;
183       }
184       if(nr < minRegions) {
185         minRegions = nr;
186       }
187       numRegions += nr;
188     }
189     if(maxRegions - minRegions < 2) {
190       // less than 2 between max and min, can't balance
191       return;
192     }
193     int min = numRegions / numServers;
194     int max = numRegions % numServers == 0 ? min : min + 1;
195 
196     for(ServerAndLoad server : servers) {
197       assertTrue(server.getLoad() <= max);
198       assertTrue(server.getLoad() >= min);
199     }
200   }
201 
202   /**
203    * Tests immediate assignment.
204    *
205    * Invariant is that all regions have an assignment.
206    *
207    * @throws Exception
208    */
209   @Test
210   public void testImmediateAssignment() throws Exception {
211     for(int [] mock : regionsAndServersMocks) {
212       LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
213       List<HRegionInfo> regions = randomRegions(mock[0]);
214       List<ServerAndLoad> servers = randomServers(mock[1], 0);
215       List<ServerName> list = getListOfServerNames(servers);
216       Map<HRegionInfo,ServerName> assignments =
217         loadBalancer.immediateAssignment(regions, list);
218       assertImmediateAssignment(regions, list, assignments);
219       returnRegions(regions);
220       returnServers(list);
221     }
222   }
223 
224   /**
225    * All regions have an assignment.
226    * @param regions
227    * @param servers
228    * @param assignments
229    */
230   private void assertImmediateAssignment(List<HRegionInfo> regions,
231       List<ServerName> servers, Map<HRegionInfo, ServerName> assignments) {
232     for(HRegionInfo region : regions) {
233       assertTrue(assignments.containsKey(region));
234     }
235   }
236 
237   /**
238    * Tests the bulk assignment used during cluster startup.
239    *
240    * Round-robin.  Should yield a balanced cluster so same invariant as the load
241    * balancer holds, all servers holding either floor(avg) or ceiling(avg).
242    *
243    * @throws Exception
244    */
245   @Test
246   public void testBulkAssignment() throws Exception {
247     for(int [] mock : regionsAndServersMocks) {
248       LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
249       List<HRegionInfo> regions = randomRegions(mock[0]);
250       List<ServerAndLoad> servers = randomServers(mock[1], 0);
251       List<ServerName> list = getListOfServerNames(servers);
252       Map<ServerName, List<HRegionInfo>> assignments =
253         loadBalancer.roundRobinAssignment(regions, list);
254       float average = (float)regions.size()/servers.size();
255       int min = (int)Math.floor(average);
256       int max = (int)Math.ceil(average);
257       if(assignments != null && !assignments.isEmpty()) {
258         for(List<HRegionInfo> regionList : assignments.values()) {
259           assertTrue(regionList.size() == min || regionList.size() == max);
260         }
261       }
262       returnRegions(regions);
263       returnServers(list);
264     }
265   }
266 
267   /**
268    * Test the cluster startup bulk assignment which attempts to retain
269    * assignment info.
270    * @throws Exception
271    */
272   @Test
273   public void testRetainAssignment() throws Exception {
274     // Test simple case where all same servers are there
275     List<ServerAndLoad> servers = randomServers(10, 10);
276     List<HRegionInfo> regions = randomRegions(100);
277     Map<HRegionInfo, ServerName> existing =
278       new TreeMap<HRegionInfo, ServerName>();
279     for (int i = 0; i < regions.size(); i++) {
280       ServerName sn = servers.get(i % servers.size()).getServerName();
281       // The old server would have had same host and port, but different
282       // start code!
283       ServerName snWithOldStartCode =
284         new ServerName(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10);
285       existing.put(regions.get(i), snWithOldStartCode);
286     }
287     List<ServerName> listOfServerNames = getListOfServerNames(servers);
288     Map<ServerName, List<HRegionInfo>> assignment =
289       loadBalancer.retainAssignment(existing, listOfServerNames);
290     assertRetainedAssignment(existing, listOfServerNames, assignment);
291 
292     // Include two new servers that were not there before
293     List<ServerAndLoad> servers2 =
294       new ArrayList<ServerAndLoad>(servers);
295     servers2.add(randomServer(10));
296     servers2.add(randomServer(10));
297     listOfServerNames = getListOfServerNames(servers2);
298     assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
299     assertRetainedAssignment(existing, listOfServerNames, assignment);
300 
301     // Remove two of the servers that were previously there
302     List<ServerAndLoad> servers3 =
303       new ArrayList<ServerAndLoad>(servers);
304     servers3.remove(0);
305     servers3.remove(0);
306     listOfServerNames = getListOfServerNames(servers3);
307     assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
308     assertRetainedAssignment(existing, listOfServerNames, assignment);
309   }
310 
311   private List<ServerName> getListOfServerNames(final List<ServerAndLoad> sals) {
312     List<ServerName> list = new ArrayList<ServerName>();
313     for (ServerAndLoad e: sals) {
314       list.add(e.getServerName());
315     }
316     return list;
317   }
318 
319   /**
320    * Asserts a valid retained assignment plan.
321    * <p>
322    * Must meet the following conditions:
323    * <ul>
324    *   <li>Every input region has an assignment, and to an online server
325    *   <li>If a region had an existing assignment to a server with the same
326    *       address a a currently online server, it will be assigned to it
327    * </ul>
328    * @param existing
329    * @param servers
330    * @param assignment
331    */
332   private void assertRetainedAssignment(
333       Map<HRegionInfo, ServerName> existing, List<ServerName> servers,
334       Map<ServerName, List<HRegionInfo>> assignment) {
335     // Verify condition 1, every region assigned, and to online server
336     Set<ServerName> onlineServerSet = new TreeSet<ServerName>(servers);
337     Set<HRegionInfo> assignedRegions = new TreeSet<HRegionInfo>();
338     for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
339       assertTrue("Region assigned to server that was not listed as online",
340           onlineServerSet.contains(a.getKey()));
341       for (HRegionInfo r : a.getValue()) assignedRegions.add(r);
342     }
343     assertEquals(existing.size(), assignedRegions.size());
344 
345     // Verify condition 2, if server had existing assignment, must have same
346     Set<String> onlineHostNames = new TreeSet<String>();
347     for (ServerName s : servers) {
348       onlineHostNames.add(s.getHostname());
349     }
350     
351     for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
352       ServerName assignedTo = a.getKey();
353       for (HRegionInfo r : a.getValue()) {
354         ServerName address = existing.get(r);
355         if (address != null && onlineHostNames.contains(address.getHostname())) {
356           // this region was prevously assigned somewhere, and that
357           // host is still around, then it should be re-assigned on the
358           // same host
359           assertEquals(address.getHostname(), assignedTo.getHostname());
360         }
361       }
362     }
363   }
364 
365   private String printStats(List<ServerAndLoad> servers) {
366     int numServers = servers.size();
367     int totalRegions = 0;
368     for(ServerAndLoad server : servers) {
369       totalRegions += server.getLoad();
370     }
371     float average = (float)totalRegions / numServers;
372     int max = (int)Math.ceil(average);
373     int min = (int)Math.floor(average);
374     return "[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]";
375   }
376 
377   private List<ServerAndLoad> convertToList(final Map<ServerName, List<HRegionInfo>> servers) {
378     List<ServerAndLoad> list =
379       new ArrayList<ServerAndLoad>(servers.size());
380     for (Map.Entry<ServerName, List<HRegionInfo>> e: servers.entrySet()) {
381       list.add(new ServerAndLoad(e.getKey(), e.getValue().size()));
382     }
383     return list;
384   }
385 
386   private String printMock(List<ServerAndLoad> balancedCluster) {
387     SortedSet<ServerAndLoad> sorted =
388       new TreeSet<ServerAndLoad>(balancedCluster);
389     ServerAndLoad [] arr =
390       sorted.toArray(new ServerAndLoad[sorted.size()]);
391     StringBuilder sb = new StringBuilder(sorted.size() * 4 + 4);
392     sb.append("{ ");
393     for(int i = 0; i < arr.length; i++) {
394       if (i != 0) {
395         sb.append(" , ");
396       }
397       sb.append(arr[i].getLoad());
398     }
399     sb.append(" }");
400     return sb.toString();
401   }
402 
403   /**
404    * This assumes the RegionPlan HSI instances are the same ones in the map, so
405    * actually no need to even pass in the map, but I think it's clearer.
406    * @param list
407    * @param plans
408    * @return
409    */
410   private List<ServerAndLoad> reconcile(List<ServerAndLoad> list,
411       List<RegionPlan> plans) {
412     List<ServerAndLoad> result =
413       new ArrayList<ServerAndLoad>(list.size());
414     if (plans == null) return result;
415     Map<ServerName, ServerAndLoad> map =
416       new HashMap<ServerName, ServerAndLoad>(list.size());
417     for (RegionPlan plan : plans) {
418       ServerName source = plan.getSource();
419       updateLoad(map, source, -1);
420       ServerName destination = plan.getDestination();
421       updateLoad(map, destination, +1);
422     }
423     result.clear();
424     result.addAll(map.values());
425     return result;
426   }
427 
428   private void updateLoad(Map<ServerName, ServerAndLoad> map,
429       final ServerName sn, final int diff) {
430     ServerAndLoad sal = map.get(sn);
431     if (sal == null) return;
432     sal = new ServerAndLoad(sn, sal.getLoad() + diff);
433     map.put(sn, sal);
434   }
435 
436   private Map<ServerName, List<HRegionInfo>> mockClusterServers(
437       int [] mockCluster) {
438     int numServers = mockCluster.length;
439     Map<ServerName, List<HRegionInfo>> servers =
440       new TreeMap<ServerName, List<HRegionInfo>>();
441     for(int i = 0; i < numServers; i++) {
442       int numRegions = mockCluster[i];
443       ServerAndLoad sal = randomServer(0);
444       List<HRegionInfo> regions = randomRegions(numRegions);
445       servers.put(sal.getServerName(), regions);
446     }
447     return servers;
448   }
449 
450   private Queue<HRegionInfo> regionQueue = new LinkedList<HRegionInfo>();
451   static int regionId = 0;
452 
453   private List<HRegionInfo> randomRegions(int numRegions) {
454     List<HRegionInfo> regions = new ArrayList<HRegionInfo>(numRegions);
455     byte [] start = new byte[16];
456     byte [] end = new byte[16];
457     rand.nextBytes(start);
458     rand.nextBytes(end);
459     for(int i=0;i<numRegions;i++) {
460       if(!regionQueue.isEmpty()) {
461         regions.add(regionQueue.poll());
462         continue;
463       }
464       Bytes.putInt(start, 0, numRegions << 1);
465       Bytes.putInt(end, 0, (numRegions << 1) + 1);
466       HRegionInfo hri = new HRegionInfo(
467           Bytes.toBytes("table" + i), start, end,
468           false, regionId++);
469       regions.add(hri);
470     }
471     return regions;
472   }
473 
474   private void returnRegions(List<HRegionInfo> regions) {
475     regionQueue.addAll(regions);
476   }
477 
478   private Queue<ServerName> serverQueue = new LinkedList<ServerName>();
479 
480   private ServerAndLoad randomServer(final int numRegionsPerServer) {
481     if (!this.serverQueue.isEmpty()) {
482       ServerName sn = this.serverQueue.poll();
483       return new ServerAndLoad(sn, numRegionsPerServer);
484     }
485     String host = "server" + rand.nextInt(100000);
486     int port = rand.nextInt(60000);
487     long startCode = rand.nextLong();
488     ServerName sn = new ServerName(host, port, startCode);
489     return new ServerAndLoad(sn, numRegionsPerServer);
490   }
491 
492   private List<ServerAndLoad> randomServers(int numServers, int numRegionsPerServer) {
493     List<ServerAndLoad> servers =
494       new ArrayList<ServerAndLoad>(numServers);
495     for (int i = 0; i < numServers; i++) {
496       servers.add(randomServer(numRegionsPerServer));
497     }
498     return servers;
499   }
500 
501   private void returnServer(ServerName server) {
502     serverQueue.add(server);
503   }
504 
505   private void returnServers(List<ServerName> servers) {
506     this.serverQueue.addAll(servers);
507   }
508 
509   @org.junit.Rule
510   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
511     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
512 }
513