1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22 import java.io.IOException;
23 import java.util.List;
24 import java.util.ArrayList;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28
29 import org.apache.hadoop.hbase.client.HTable;
30 import org.apache.hadoop.hbase.client.Put;
31
32 import org.apache.hadoop.hbase.regionserver.HRegionServer;
33 import org.apache.hadoop.hbase.regionserver.HRegion;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.apache.hadoop.hbase.util.JVMClusterUtil;
36
37
38
39
40 public class TestRegionRebalancing extends HBaseClusterTestCase {
41 final Log LOG = LogFactory.getLog(this.getClass().getName());
42 HTable table;
43
44 HTableDescriptor desc;
45
46 final byte[] FIVE_HUNDRED_KBYTES;
47
48 final byte [] FAMILY_NAME = Bytes.toBytes("col");
49
50
51 public TestRegionRebalancing() {
52 super(1);
53 FIVE_HUNDRED_KBYTES = new byte[500 * 1024];
54 for (int i = 0; i < 500 * 1024; i++) {
55 FIVE_HUNDRED_KBYTES[i] = 'x';
56 }
57
58 desc = new HTableDescriptor("test");
59 desc.addFamily(new HColumnDescriptor(FAMILY_NAME));
60 }
61
62
63
64
65 @Override
66 public void preHBaseClusterSetup() throws IOException {
67
68 List<byte []> startKeys = new ArrayList<byte []>();
69 startKeys.add(null);
70 for (int i = 10; i < 29; i++) {
71 startKeys.add(Bytes.toBytes("row_" + i));
72 }
73 startKeys.add(null);
74 LOG.info(startKeys.size() + " start keys generated");
75
76 List<HRegion> regions = new ArrayList<HRegion>();
77 for (int i = 0; i < 20; i++) {
78 regions.add(createAregion(startKeys.get(i), startKeys.get(i+1)));
79 }
80
81
82
83
84 createRootAndMetaRegions();
85 for (HRegion region : regions) {
86 HRegion.addRegionToMETA(meta, region);
87 }
88 closeRootAndMeta();
89 }
90
91
92
93
94
95
96 public void testRebalancing() throws IOException {
97 table = new HTable(conf, "test");
98 assertEquals("Test table should have 20 regions",
99 20, table.getStartKeys().length);
100
101
102 assertRegionsAreBalanced();
103
104 LOG.debug("Adding 2nd region server.");
105
106 cluster.startRegionServer();
107 assertRegionsAreBalanced();
108
109
110 LOG.debug("Adding 3rd region server.");
111 cluster.startRegionServer();
112 assertRegionsAreBalanced();
113
114
115 LOG.debug("Killing the 3rd region server.");
116 cluster.stopRegionServer(2, false);
117 cluster.waitOnRegionServer(2);
118 assertRegionsAreBalanced();
119
120
121 LOG.debug("Adding 3rd region server");
122 cluster.startRegionServer();
123 LOG.debug("Adding 4th region server");
124 cluster.startRegionServer();
125 assertRegionsAreBalanced();
126
127 for (int i = 0; i < 6; i++){
128 LOG.debug("Adding " + (i + 5) + "th region server");
129 cluster.startRegionServer();
130 }
131 assertRegionsAreBalanced();
132 }
133
134
135 private int getRegionCount() {
136 int total = 0;
137 for (HRegionServer server : getOnlineRegionServers()) {
138 total += server.getOnlineRegions().size();
139 }
140 return total;
141 }
142
143
144
145
146
147
148 private void assertRegionsAreBalanced() {
149 boolean success = false;
150 float slop = conf.getFloat("hbase.regions.slop", (float)0.1);
151 if (slop <= 0) slop = 1;
152
153 for (int i = 0; i < 5; i++) {
154 success = true;
155
156 waitForAllRegionsAssigned();
157
158 int regionCount = getRegionCount();
159 List<HRegionServer> servers = getOnlineRegionServers();
160 double avg = cluster.getMaster().getAverageLoad();
161 int avgLoadPlusSlop = (int)Math.ceil(avg * (1 + slop));
162 int avgLoadMinusSlop = (int)Math.floor(avg * (1 - slop)) - 1;
163 LOG.debug("There are " + servers.size() + " servers and " + regionCount
164 + " regions. Load Average: " + avg + " low border: " + avgLoadMinusSlop
165 + ", up border: " + avgLoadPlusSlop + "; attempt: " + i);
166
167 for (HRegionServer server : servers) {
168 int serverLoad = server.getOnlineRegions().size();
169 LOG.debug(server.hashCode() + " Avg: " + avg + " actual: " + serverLoad);
170 if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop
171 && serverLoad >= avgLoadMinusSlop)) {
172 LOG.debug(server.hashCode() + " Isn't balanced!!! Avg: " + avg +
173 " actual: " + serverLoad + " slop: " + slop);
174 success = false;
175 }
176 }
177
178 if (!success) {
179
180
181 try {
182 Thread.sleep(10000);
183 } catch (InterruptedException e) {}
184
185 continue;
186 }
187
188
189 return;
190 }
191
192
193 fail("After 5 attempts, region assignments were not balanced.");
194 }
195
196 private List<HRegionServer> getOnlineRegionServers() {
197 List<HRegionServer> list = new ArrayList<HRegionServer>();
198 for (JVMClusterUtil.RegionServerThread rst : cluster.getRegionServerThreads()) {
199 if (rst.getRegionServer().isOnline()) {
200 list.add(rst.getRegionServer());
201 }
202 }
203 return list;
204 }
205
206
207
208
209 private void waitForAllRegionsAssigned() {
210 while (getRegionCount() < 22) {
211
212 LOG.debug("Waiting for there to be 22 regions, but there are " + getRegionCount() + " right now.");
213 try {
214 Thread.sleep(1000);
215 } catch (InterruptedException e) {}
216 }
217 }
218
219
220
221
222
223 private HRegion createAregion(byte [] startKey, byte [] endKey)
224 throws IOException {
225 HRegion region = createNewHRegion(desc, startKey, endKey);
226 byte [] keyToWrite = startKey == null ? Bytes.toBytes("row_000") : startKey;
227 Put put = new Put(keyToWrite);
228 put.add(FAMILY_NAME, null, Bytes.toBytes("test"));
229 region.put(put);
230 region.close();
231 region.getLog().closeAndDelete();
232 return region;
233 }
234 }