1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.net.InetSocketAddress;
27 import java.util.ArrayList;
28 import java.util.Collection;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Set;
33 import java.util.concurrent.atomic.AtomicInteger;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.HBaseIOException;
39 import org.apache.hadoop.hbase.TableName;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.MediumTests;
46 import org.apache.hadoop.hbase.MiniHBaseCluster;
47 import org.apache.hadoop.hbase.NamespaceDescriptor;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.client.HBaseAdmin;
50 import org.apache.hadoop.hbase.client.HTable;
51 import org.apache.hadoop.hbase.client.MetaScanner;
52 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
53 import org.apache.hadoop.hbase.client.Result;
54 import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
55 import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
56 import org.apache.hadoop.hbase.master.balancer.FavoredNodes.Position;
57 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
58 import org.apache.hadoop.hbase.regionserver.HRegion;
59 import org.apache.hadoop.hbase.regionserver.HRegionServer;
60 import org.apache.hadoop.hbase.util.Bytes;
61 import org.junit.AfterClass;
62 import org.junit.BeforeClass;
63 import org.junit.Test;
64 import org.junit.experimental.categories.Category;
65
66 @Category(MediumTests.class)
67 public class TestRegionPlacement {
68 final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
69 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
70 private final static int SLAVES = 10;
71 private static HBaseAdmin admin;
72 private static Position[] positions = Position.values();
73 private int REGION_NUM = 10;
74 private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
75 new HashMap<HRegionInfo, ServerName[]>();
76 private final static int PRIMARY = Position.PRIMARY.ordinal();
77 private final static int SECONDARY = Position.SECONDARY.ordinal();
78 private final static int TERTIARY = Position.TERTIARY.ordinal();
79
80 @BeforeClass
81 public static void setupBeforeClass() throws Exception {
82 Configuration conf = TEST_UTIL.getConfiguration();
83
84 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
85 FavoredNodeLoadBalancer.class, LoadBalancer.class);
86 conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
87 TEST_UTIL.startMiniCluster(SLAVES);
88 admin = new HBaseAdmin(conf);
89 }
90
91 @AfterClass
92 public static void tearDownAfterClass() throws Exception {
93 TEST_UTIL.shutdownMiniCluster();
94 }
95
96 @Test
97 public void testFavoredNodesPresentForRoundRobinAssignment() throws HBaseIOException {
98 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
99 balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
100 List<ServerName> servers = new ArrayList<ServerName>();
101 for (int i = 0; i < SLAVES; i++) {
102 ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
103 servers.add(server);
104 }
105 List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
106 HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
107 regions.add(region);
108 Map<ServerName,List<HRegionInfo>> assignmentMap = balancer.roundRobinAssignment(regions,
109 servers);
110 Set<ServerName> serverBefore = assignmentMap.keySet();
111 List<ServerName> favoredNodesBefore =
112 ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
113 assertTrue(favoredNodesBefore.size() == 3);
114
115 assertTrue(ServerName.isSameHostnameAndPort(serverBefore.iterator().next(),
116 favoredNodesBefore.get(PRIMARY)));
117
118 List<ServerName> removedServers = removeMatchingServers(serverBefore, servers);
119
120 assignmentMap = balancer.roundRobinAssignment(regions, servers);
121 List<ServerName> favoredNodesAfter =
122 ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
123 assertTrue(favoredNodesAfter.size() == 3);
124
125
126
127 assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
128 Set<ServerName> serverAfter = assignmentMap.keySet();
129
130
131 assertTrue(ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
132 favoredNodesBefore.get(SECONDARY)) ||
133 ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
134 favoredNodesBefore.get(TERTIARY)));
135
136
137 servers.addAll(removedServers);
138
139
140 assignmentMap = balancer.roundRobinAssignment(regions, servers);
141 Set<ServerName> serverWithPrimary = assignmentMap.keySet();
142 assertTrue(serverBefore.containsAll(serverWithPrimary));
143
144
145 removeMatchingServers(favoredNodesAfter, servers);
146
147 assignmentMap = balancer.roundRobinAssignment(regions, servers);
148 List<ServerName> favoredNodesNow =
149 ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
150 assertTrue(favoredNodesNow.size() == 3);
151 assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
152 !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
153 !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
154 }
155
156 @Test
157 public void testFavoredNodesPresentForRandomAssignment() throws HBaseIOException {
158 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
159 balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
160 List<ServerName> servers = new ArrayList<ServerName>();
161 for (int i = 0; i < SLAVES; i++) {
162 ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
163 servers.add(server);
164 }
165 List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
166 HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
167 regions.add(region);
168 ServerName serverBefore = balancer.randomAssignment(region, servers);
169 List<ServerName> favoredNodesBefore =
170 ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
171 assertTrue(favoredNodesBefore.size() == 3);
172
173 assertTrue(ServerName.isSameHostnameAndPort(serverBefore,favoredNodesBefore.get(PRIMARY)));
174
175 removeMatchingServers(serverBefore, servers);
176
177 ServerName serverAfter = balancer.randomAssignment(region, servers);
178 List<ServerName> favoredNodesAfter =
179 ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
180 assertTrue(favoredNodesAfter.size() == 3);
181
182
183
184 assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
185
186
187 assertTrue(ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(SECONDARY)) ||
188 ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(TERTIARY)));
189
190 removeMatchingServers(favoredNodesAfter, servers);
191
192 balancer.randomAssignment(region, servers);
193 List<ServerName> favoredNodesNow =
194 ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
195 assertTrue(favoredNodesNow.size() == 3);
196 assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
197 !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
198 !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
199 }
200
201 @Test(timeout = 180000)
202 public void testRegionPlacement() throws Exception {
203
204 createTable("testRegionAssignment", REGION_NUM);
205
206 TEST_UTIL.waitTableAvailable(Bytes.toBytes("testRegionAssignment"));
207
208
209
210 countRegionOnPrimaryRS(REGION_NUM);
211
212
213 verifyRegionServerUpdated();
214 }
215
216 private List<ServerName> removeMatchingServers(ServerName serverWithoutStartCode,
217 List<ServerName> servers) {
218 List<ServerName> serversToRemove = new ArrayList<ServerName>();
219 for (ServerName s : servers) {
220 if (ServerName.isSameHostnameAndPort(s, serverWithoutStartCode)) {
221 serversToRemove.add(s);
222 }
223 }
224 servers.removeAll(serversToRemove);
225 return serversToRemove;
226 }
227
228 private List<ServerName> removeMatchingServers(Collection<ServerName> serversWithoutStartCode,
229 List<ServerName> servers) {
230 List<ServerName> serversToRemove = new ArrayList<ServerName>();
231 for (ServerName s : serversWithoutStartCode) {
232 serversToRemove.addAll(removeMatchingServers(s, servers));
233 }
234 return serversToRemove;
235 }
236
237
238
239
240
241
242
243 private void countRegionOnPrimaryRS(int expectedNum)
244 throws IOException {
245 int lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
246 assertEquals("Only " + expectedNum + " of user regions running " +
247 "on the primary region server", expectedNum ,
248 lastRegionOnPrimaryRSCount);
249 }
250
251
252
253
254
255
256
257 private void verifyRegionServerUpdated() throws IOException {
258
259 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
260 for (int i = 0; i < SLAVES; i++) {
261 HRegionServer rs = cluster.getRegionServer(i);
262 for (HRegion region: rs.getOnlineRegions(
263 TableName.valueOf("testRegionAssignment"))) {
264 InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
265 region.getRegionInfo().getEncodedName());
266 ServerName[] favoredServerList = favoredNodesAssignmentPlan.get(region.getRegionInfo());
267
268
269
270 if (favoredServerList == null) {
271 HTableDescriptor desc = region.getTableDesc();
272
273 assertNull(favoredSocketAddress);
274 assertTrue("User region " +
275 region.getTableDesc().getTableName() +
276 " should have favored nodes",
277 (desc.isRootRegion() || desc.isMetaRegion()));
278 } else {
279
280
281 assertTrue(favoredSocketAddress.length == favoredServerList.length);
282 assertTrue(favoredServerList.length > 0);
283 for (int j = 0; j < favoredServerList.length; j++) {
284 InetSocketAddress addrFromRS = favoredSocketAddress[j];
285 InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
286 favoredServerList[j].getHostname(), favoredServerList[j].getPort());
287
288 assertNotNull(addrFromRS);
289 assertNotNull(addrFromPlan);
290 assertTrue("Region server " + rs.getServerName().getHostAndPort()
291 + " has the " + positions[j] +
292 " for region " + region.getRegionNameAsString() + " is " +
293 addrFromRS + " which is inconsistent with the plan "
294 + addrFromPlan, addrFromRS.equals(addrFromPlan));
295 }
296 }
297 }
298 }
299 }
300
301
302
303
304
305
306
307
308
309 private int getNumRegionisOnPrimaryRS() throws IOException {
310 final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
311 final AtomicInteger totalRegionNum = new AtomicInteger(0);
312 LOG.info("The start of region placement verification");
313 MetaScannerVisitor visitor = new MetaScannerVisitor() {
314 public boolean processRow(Result result) throws IOException {
315 try {
316 HRegionInfo info = MetaScanner.getHRegionInfo(result);
317 if(info.getTableName().getNamespaceAsString()
318 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
319 return true;
320 }
321 byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
322 HConstants.SERVER_QUALIFIER);
323 byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
324 FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
325
326 ServerName[] favoredServerList =
327 FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
328 favoredNodesAssignmentPlan.put(info, favoredServerList);
329
330 Position[] positions = Position.values();
331 if (info != null) {
332 totalRegionNum.incrementAndGet();
333 if (server != null) {
334 ServerName serverName =
335 new ServerName(Bytes.toString(server), -1);
336 if (favoredNodes != null) {
337 String placement = "[NOT FAVORED NODE]";
338 for (int i = 0; i < favoredServerList.length; i++) {
339 if (favoredServerList[i].equals(serverName)) {
340 placement = positions[i].toString();
341 if (i == Position.PRIMARY.ordinal()) {
342 regionOnPrimaryNum.incrementAndGet();
343 }
344 break;
345 }
346 }
347 LOG.info(info.getRegionNameAsString() + " on " +
348 serverName + " " + placement);
349 } else {
350 LOG.info(info.getRegionNameAsString() + " running on " +
351 serverName + " but there is no favored region server");
352 }
353 } else {
354 LOG.info(info.getRegionNameAsString() +
355 " not assigned to any server");
356 }
357 }
358 return true;
359 } catch (RuntimeException e) {
360 LOG.error("Result=" + result);
361 throw e;
362 }
363 }
364
365 @Override
366 public void close() throws IOException {}
367 };
368 MetaScanner.metaScan(TEST_UTIL.getConfiguration(), visitor);
369 LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
370 totalRegionNum.intValue() + " regions running on the primary" +
371 " region servers" );
372 return regionOnPrimaryNum.intValue() ;
373 }
374
375
376
377
378
379
380
381
382 private static void createTable(String table, int regionNum)
383 throws IOException {
384 byte[] tableName = Bytes.toBytes(table);
385 int expectedRegions = regionNum;
386 byte[][] splitKeys = new byte[expectedRegions - 1][];
387 for (int i = 1; i < expectedRegions; i++) {
388 byte splitKey = (byte) i;
389 splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
390 }
391
392 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
393 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
394 admin.createTable(desc, splitKeys);
395
396 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
397 Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
398 assertEquals("Tried to create " + expectedRegions + " regions "
399 + "but only found " + regions.size(), expectedRegions, regions.size());
400 }
401 }