1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.fail;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.hbase.client.Admin;
27 import org.apache.hadoop.hbase.client.Connection;
28 import org.apache.hadoop.hbase.client.ConnectionFactory;
29 import org.apache.hadoop.hbase.client.HBaseAdmin;
30 import org.apache.hadoop.hbase.client.HTable;
31 import org.apache.hadoop.hbase.client.RegionLocator;
32 import org.apache.hadoop.hbase.master.RegionStates;
33 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
34 import org.apache.hadoop.hbase.regionserver.HRegionServer;
35 import org.apache.hadoop.hbase.testclassification.LargeTests;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.hbase.util.JVMClusterUtil;
38 import org.apache.hadoop.hbase.util.Threads;
39 import org.junit.After;
40 import org.junit.Before;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43 import org.junit.runner.RunWith;
44 import org.junit.runners.Parameterized;
45 import org.junit.runners.Parameterized.Parameters;
46
47 import java.io.IOException;
48 import java.util.ArrayList;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.List;
52
53
54
55
56 @Category(LargeTests.class)
57 @RunWith(value = Parameterized.class)
58 public class TestRegionRebalancing {
59
60 @Parameters
61 public static Collection<Object[]> data() {
62 Object[][] balancers =
63 new String[][] { { "org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer" },
64 { "org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer" } };
65 return Arrays.asList(balancers);
66 }
67
68 private static final byte[] FAMILY_NAME = Bytes.toBytes("col");
69 public static final Log LOG = LogFactory.getLog(TestRegionRebalancing.class);
70 private final HBaseTestingUtility UTIL = new HBaseTestingUtility();
71 private RegionLocator table;
72 private HTableDescriptor desc;
73 private String balancerName;
74
75 public TestRegionRebalancing(String balancerName) {
76 this.balancerName = balancerName;
77
78 }
79
80 @After
81 public void after() throws Exception {
82 UTIL.shutdownMiniCluster();
83 }
84
85 @Before
86 public void before() throws Exception {
87 UTIL.getConfiguration().set("hbase.master.loadbalancer.class", this.balancerName);
88 UTIL.startMiniCluster(1);
89 this.desc = new HTableDescriptor(TableName.valueOf("test"));
90 this.desc.addFamily(new HColumnDescriptor(FAMILY_NAME));
91 }
92
93
94
95
96
97
98
99 @Test (timeout=300000)
100 @SuppressWarnings("deprecation")
101 public void testRebalanceOnRegionServerNumberChange()
102 throws IOException, InterruptedException {
103 Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
104 Admin admin = connection.getAdmin();
105 admin.createTable(this.desc, Arrays.copyOfRange(HBaseTestingUtility.KEYS,
106 1, HBaseTestingUtility.KEYS.length));
107 this.table = new HTable(UTIL.getConfiguration(), this.desc.getTableName());
108
109 MetaTableAccessor.fullScanMetaAndPrint(admin.getConnection());
110
111 assertEquals("Test table should have right number of regions",
112 HBaseTestingUtility.KEYS.length,
113 this.table.getStartKeys().length);
114
115
116 assertRegionsAreBalanced();
117
118
119 LOG.info("Started second server=" +
120 UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName());
121 UTIL.getHBaseCluster().getMaster().balance();
122 assertRegionsAreBalanced();
123
124
125 assert(UTIL.getHBaseCluster().getMaster().balance() == true);
126
127
128
129 LOG.info("Started third server=" +
130 UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName());
131 assert(UTIL.getHBaseCluster().getMaster().balance() == true);
132 assertRegionsAreBalanced();
133
134
135 LOG.info("Stopped third server=" + UTIL.getHBaseCluster().stopRegionServer(2, false));
136 UTIL.getHBaseCluster().waitOnRegionServer(2);
137 UTIL.getHBaseCluster().getMaster().balance();
138 assertRegionsAreBalanced();
139
140
141 LOG.info("Readding third server=" +
142 UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName());
143 LOG.info("Added fourth server=" +
144 UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName());
145 assert(UTIL.getHBaseCluster().getMaster().balance() == true);
146 assertRegionsAreBalanced();
147
148 for (int i = 0; i < 6; i++){
149 LOG.info("Adding " + (i + 5) + "th region server");
150 UTIL.getHBaseCluster().startRegionServer();
151 }
152 assert(UTIL.getHBaseCluster().getMaster().balance() == true);
153 assertRegionsAreBalanced();
154 table.close();
155 admin.close();
156 }
157
158
159
160
161
162
163 private void assertRegionsAreBalanced() throws IOException {
164
165
166 boolean success = false;
167 float slop = (float)UTIL.getConfiguration().getFloat("hbase.regions.slop", 0.1f);
168 if (slop <= 0) slop = 1;
169
170 for (int i = 0; i < 5; i++) {
171 success = true;
172
173 waitForAllRegionsAssigned();
174
175 long regionCount = UTIL.getMiniHBaseCluster().countServedRegions();
176 List<HRegionServer> servers = getOnlineRegionServers();
177 double avg = UTIL.getHBaseCluster().getMaster().getAverageLoad();
178 int avgLoadPlusSlop = (int)Math.ceil(avg * (1 + slop));
179 int avgLoadMinusSlop = (int)Math.floor(avg * (1 - slop)) - 1;
180 LOG.debug("There are " + servers.size() + " servers and " + regionCount
181 + " regions. Load Average: " + avg + " low border: " + avgLoadMinusSlop
182 + ", up border: " + avgLoadPlusSlop + "; attempt: " + i);
183
184 for (HRegionServer server : servers) {
185 int serverLoad =
186 ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
187 LOG.debug(server.getServerName() + " Avg: " + avg + " actual: " + serverLoad);
188 if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
189 && serverLoad >= avgLoadMinusSlop)) {
190 for (HRegionInfo hri :
191 ProtobufUtil.getOnlineRegions(server.getRSRpcServices())) {
192 if (hri.isMetaRegion()) serverLoad--;
193
194 }
195 if (!(serverLoad <= avgLoadPlusSlop && serverLoad >= avgLoadMinusSlop)) {
196 LOG.debug(server.getServerName() + " Isn't balanced!!! Avg: " + avg +
197 " actual: " + serverLoad + " slop: " + slop);
198 success = false;
199 break;
200 }
201 }
202 }
203
204 if (!success) {
205
206
207 try {
208 Thread.sleep(10000);
209 } catch (InterruptedException e) {}
210
211 UTIL.getHBaseCluster().getMaster().balance();
212 continue;
213 }
214
215
216 return;
217 }
218
219
220 fail("After 5 attempts, region assignments were not balanced.");
221 }
222
223 private List<HRegionServer> getOnlineRegionServers() {
224 List<HRegionServer> list = new ArrayList<HRegionServer>();
225 for (JVMClusterUtil.RegionServerThread rst :
226 UTIL.getHBaseCluster().getRegionServerThreads()) {
227 if (rst.getRegionServer().isOnline()) {
228 list.add(rst.getRegionServer());
229 }
230 }
231 return list;
232 }
233
234
235
236
237 private void waitForAllRegionsAssigned() throws IOException {
238 int totalRegions = HBaseTestingUtility.KEYS.length;
239 while (UTIL.getMiniHBaseCluster().countServedRegions() < totalRegions) {
240
241 LOG.debug("Waiting for there to be "+ totalRegions +" regions, but there are "
242 + UTIL.getMiniHBaseCluster().countServedRegions() + " right now.");
243 try {
244 Thread.sleep(200);
245 } catch (InterruptedException e) {}
246 }
247 RegionStates regionStates = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
248 while (!regionStates.getRegionsInTransition().isEmpty()) {
249 Threads.sleep(100);
250 }
251 }
252
253 }
254