1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.net.InetSocketAddress;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.HashMap;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Random;
35 import java.util.concurrent.atomic.AtomicInteger;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.MiniHBaseCluster;
47 import org.apache.hadoop.hbase.NamespaceDescriptor;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.TableName;
50 import org.apache.hadoop.hbase.client.Admin;
51 import org.apache.hadoop.hbase.client.Connection;
52 import org.apache.hadoop.hbase.client.ConnectionFactory;
53 import org.apache.hadoop.hbase.client.HBaseAdmin;
54 import org.apache.hadoop.hbase.client.HTable;
55 import org.apache.hadoop.hbase.client.MetaScanner;
56 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
57 import org.apache.hadoop.hbase.client.Result;
58 import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
59 import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
60 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan;
61 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
62 import org.apache.hadoop.hbase.regionserver.HRegion;
63 import org.apache.hadoop.hbase.regionserver.HRegionServer;
64 import org.apache.hadoop.hbase.util.Bytes;
65 import org.apache.hadoop.hbase.util.Pair;
66 import org.apache.zookeeper.KeeperException;
67 import org.junit.AfterClass;
68 import org.junit.BeforeClass;
69 import org.junit.Test;
70 import org.junit.experimental.categories.Category;
71
72
73 @Category(MediumTests.class)
74 public class TestRegionPlacement {
75 final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
76 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
77 private final static int SLAVES = 10;
78 private static Connection CONNECTION;
79 private static Admin admin;
80 private static RegionPlacementMaintainer rp;
81 private static Position[] positions = Position.values();
82 private int lastRegionOnPrimaryRSCount = 0;
83 private int REGION_NUM = 10;
84 private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
85 new HashMap<HRegionInfo, ServerName[]>();
86
87 @BeforeClass
88 public static void setupBeforeClass() throws Exception {
89 Configuration conf = TEST_UTIL.getConfiguration();
90
91 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
92 FavoredNodeLoadBalancer.class, LoadBalancer.class);
93 conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
94 TEST_UTIL.startMiniCluster(SLAVES);
95 CONNECTION = TEST_UTIL.getConnection();
96 admin = CONNECTION.getAdmin();
97 rp = new RegionPlacementMaintainer(conf);
98 }
99
100 @AfterClass
101 public static void tearDownAfterClass() throws Exception {
102 TEST_UTIL.shutdownMiniCluster();
103 }
104
105 @Test
106 public void testRegionPlacement() throws Exception {
107 String tableStr = "testRegionAssignment";
108 TableName table = TableName.valueOf(tableStr);
109
110 createTable(table, REGION_NUM);
111
112 TEST_UTIL.waitTableAvailable(table);
113
114
115
116 verifyRegionOnPrimaryRS(REGION_NUM);
117
118 FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
119
120 verifyRegionServerUpdated(currentPlan);
121
122
123
124
125
126
127
128
129 FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
130 FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
131
132 rp.updateAssignmentPlan(shuffledPlan);
133
134
135
136 verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
137
138
139
140 shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
141 FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
142
143
144 rp.updateAssignmentPlan(shuffledPlan);
145
146 verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
147
148
149 RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
150
151 rp.setTargetTableName(new String[]{tableStr});
152 List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
153 AssignmentVerificationReport report = reports.get(0);
154 assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
155 assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
156 assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
157 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
158 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
159 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
160 assertTrue(report.getUnassignedRegions().size() == 0);
161
162
163 killRandomServerAndVerifyAssignment();
164
165
166 reports = rp.verifyRegionPlacement(false);
167 report = reports.get(0);
168 assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
169 assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
170 assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
171 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
172 assertTrue("secondary " +
173 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
174 + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
175 (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
176 || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
177 assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
178 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
179 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
180 RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
181 }
182
183 private void killRandomServerAndVerifyAssignment()
184 throws IOException, InterruptedException, KeeperException {
185 ServerName serverToKill = null;
186 int killIndex = 0;
187 Random random = new Random(System.currentTimeMillis());
188 ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
189 LOG.debug("Server holding meta " + metaServer);
190 boolean isNamespaceServer = false;
191 do {
192
193 killIndex = random.nextInt(SLAVES);
194 serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
195 Collection<HRegion> regs =
196 TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
197 isNamespaceServer = false;
198 for (HRegion r : regs) {
199 if (r.getRegionInfo().getTable().getNamespaceAsString()
200 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
201 isNamespaceServer = true;
202 break;
203 }
204 }
205 } while (ServerName.isSameHostnameAndPort(metaServer, serverToKill) || isNamespaceServer ||
206 TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
207 LOG.debug("Stopping RS " + serverToKill);
208 Map<HRegionInfo, Pair<ServerName, ServerName>> regionsToVerify =
209 new HashMap<HRegionInfo, Pair<ServerName, ServerName>>();
210
211 for (Map.Entry<HRegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
212 ServerName s = entry.getValue()[0];
213 if (ServerName.isSameHostnameAndPort(s, serverToKill)) {
214 regionsToVerify.put(entry.getKey(), new Pair<ServerName, ServerName>(
215 entry.getValue()[1], entry.getValue()[2]));
216 LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
217 entry.getValue()[1] + " " + entry.getValue()[2]);
218 }
219 }
220 int orig = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
221 TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
222 TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
223 int curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
224 while (curr - orig < regionsToVerify.size()) {
225 LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
226 " Current #regions " + curr + " Original #regions " + orig);
227 Thread.sleep(200);
228 curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
229 }
230
231 for (Map.Entry<HRegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
232 ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
233 .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
234 Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
235 LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
236 " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
237 + "/" + secondaryTertiaryServers.getSecond());
238 if (!(ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getFirst())||
239 ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getSecond()))){
240 fail("Region " + entry.getKey() + " not present on any of the expected servers");
241 }
242 }
243
244 TEST_UTIL.getHBaseCluster().startRegionServer();
245 }
246
247
248
249
250 @Test
251 public void testRandomizedMatrix() {
252 int rows = 100;
253 int cols = 100;
254 float[][] matrix = new float[rows][cols];
255 Random random = new Random();
256 for (int i = 0; i < rows; i++) {
257 for (int j = 0; j < cols; j++) {
258 matrix[i][j] = random.nextFloat();
259 }
260 }
261
262
263 RegionPlacementMaintainer.RandomizedMatrix rm =
264 new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
265 float[][] transformed = rm.transform(matrix);
266 float[][] invertedTransformed = rm.invert(transformed);
267 for (int i = 0; i < rows; i++) {
268 for (int j = 0; j < cols; j++) {
269 if (matrix[i][j] != invertedTransformed[i][j]) {
270 throw new RuntimeException();
271 }
272 }
273 }
274
275
276
277 int[] transformedIndices = new int[rows];
278 for (int i = 0; i < rows; i++) {
279 transformedIndices[i] = random.nextInt(cols);
280 }
281 int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
282 float[] transformedValues = new float[rows];
283 float[] invertedTransformedValues = new float[rows];
284 for (int i = 0; i < rows; i++) {
285 transformedValues[i] = transformed[i][transformedIndices[i]];
286 invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
287 }
288 Arrays.sort(transformedValues);
289 Arrays.sort(invertedTransformedValues);
290 if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
291 throw new RuntimeException();
292 }
293 }
294
295
296
297
298
299
300
301
302 private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
303 FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) {
304 FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
305
306 for (Map.Entry<HRegionInfo, List<ServerName>> entry :
307 plan.getAssignmentMap().entrySet()) {
308 HRegionInfo region = entry.getKey();
309
310
311 List<ServerName> shuffledServerList = new ArrayList<ServerName>();
312 shuffledServerList.addAll(entry.getValue());
313
314
315 shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
316 shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
317
318
319 shuffledPlan.updateAssignmentPlan(region, shuffledServerList);
320 }
321 return shuffledPlan;
322 }
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337 private void verifyRegionAssignment(FavoredNodesPlan plan,
338 int regionMovementNum, int numRegionsOnPrimaryRS)
339 throws InterruptedException, IOException {
340
341 verifyMETAUpdated(plan);
342
343
344 verifyRegionMovementNum(regionMovementNum);
345
346
347
348 verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
349
350
351 verifyRegionServerUpdated(plan);
352 }
353
354
355
356
357
358
359 private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
360 throws IOException {
361 FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
362 assertTrue("The assignment plan is NOT consistent with the expected plan ",
363 planFromMETA.equals(expectedPlan));
364 }
365
366
367
368
369 private void verifyRegionMovementNum(int expected)
370 throws InterruptedException, IOException {
371 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
372 HMaster m = cluster.getMaster();
373 int lastRegionOpenedCount = m.assignmentManager.getNumRegionsOpened();
374
375 m.balance();
376
377 int retry = 10;
378 long sleep = 3000;
379 int attempt = 0;
380 int currentRegionOpened, regionMovement;
381 do {
382 currentRegionOpened = m.assignmentManager.getNumRegionsOpened();
383 regionMovement= currentRegionOpened - lastRegionOpenedCount;
384 LOG.debug("There are " + regionMovement + "/" + expected +
385 " regions moved after " + attempt + " attempts");
386 Thread.sleep((++attempt) * sleep);
387 } while (regionMovement != expected && attempt <= retry);
388
389
390 lastRegionOpenedCount = currentRegionOpened;
391
392 assertEquals("There are only " + regionMovement + " instead of "
393 + expected + " region movement for " + attempt + " attempts",
394 regionMovement, expected);
395 }
396
397
398
399
400
401
402
403 private void verifyRegionOnPrimaryRS(int expectedNum)
404 throws IOException {
405 lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
406 assertEquals("Only " + expectedNum + " of user regions running " +
407 "on the primary region server", expectedNum ,
408 lastRegionOnPrimaryRSCount);
409 }
410
411
412
413
414
415
416
417 private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
418
419 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
420 for (int i = 0; i < SLAVES; i++) {
421 HRegionServer rs = cluster.getRegionServer(i);
422 for (HRegion region: rs.getOnlineRegions(
423 TableName.valueOf("testRegionAssignment"))) {
424 InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
425 region.getRegionInfo().getEncodedName());
426 List<ServerName> favoredServerList = plan.getAssignmentMap().get(region.getRegionInfo());
427
428
429
430 if (favoredServerList == null) {
431 HTableDescriptor desc = region.getTableDesc();
432
433 assertNull(favoredSocketAddress);
434 assertTrue("User region " +
435 region.getTableDesc().getTableName() +
436 " should have favored nodes",
437 (desc.isRootRegion() || desc.isMetaRegion()));
438 } else {
439
440
441 assertTrue(favoredSocketAddress.length == favoredServerList.size());
442 assertTrue(favoredServerList.size() > 0);
443 for (int j = 0; j < favoredServerList.size(); j++) {
444 InetSocketAddress addrFromRS = favoredSocketAddress[j];
445 InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
446 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
447
448 assertNotNull(addrFromRS);
449 assertNotNull(addrFromPlan);
450 assertTrue("Region server " + rs.getServerName().getHostAndPort()
451 + " has the " + positions[j] +
452 " for region " + region.getRegionNameAsString() + " is " +
453 addrFromRS + " which is inconsistent with the plan "
454 + addrFromPlan, addrFromRS.equals(addrFromPlan));
455 }
456 }
457 }
458 }
459 }
460
461
462
463
464
465
466
467
468
469 private int getNumRegionisOnPrimaryRS() throws IOException {
470 final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
471 final AtomicInteger totalRegionNum = new AtomicInteger(0);
472 LOG.info("The start of region placement verification");
473 MetaScannerVisitor visitor = new MetaScannerVisitor() {
474 public boolean processRow(Result result) throws IOException {
475 try {
476 @SuppressWarnings("deprecation")
477 HRegionInfo info = MetaScanner.getHRegionInfo(result);
478 if(info.getTable().getNamespaceAsString()
479 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
480 return true;
481 }
482 byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
483 HConstants.SERVER_QUALIFIER);
484 byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
485 FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
486
487 ServerName[] favoredServerList =
488 FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
489 favoredNodesAssignmentPlan.put(info, favoredServerList);
490
491 Position[] positions = Position.values();
492 if (info != null) {
493 totalRegionNum.incrementAndGet();
494 if (server != null) {
495 ServerName serverName =
496 ServerName.valueOf(Bytes.toString(server), -1);
497 if (favoredNodes != null) {
498 String placement = "[NOT FAVORED NODE]";
499 for (int i = 0; i < favoredServerList.length; i++) {
500 if (favoredServerList[i].equals(serverName)) {
501 placement = positions[i].toString();
502 if (i == Position.PRIMARY.ordinal()) {
503 regionOnPrimaryNum.incrementAndGet();
504 }
505 break;
506 }
507 }
508 LOG.info(info.getRegionNameAsString() + " on " +
509 serverName + " " + placement);
510 } else {
511 LOG.info(info.getRegionNameAsString() + " running on " +
512 serverName + " but there is no favored region server");
513 }
514 } else {
515 LOG.info(info.getRegionNameAsString() +
516 " not assigned to any server");
517 }
518 }
519 return true;
520 } catch (RuntimeException e) {
521 LOG.error("Result=" + result);
522 throw e;
523 }
524 }
525
526 @Override
527 public void close() throws IOException {}
528 };
529 MetaScanner.metaScan(CONNECTION, visitor);
530 LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
531 totalRegionNum.intValue() + " regions running on the primary" +
532 " region servers" );
533 return regionOnPrimaryNum.intValue() ;
534 }
535
536
537
538
539
540
541
542
543 private static void createTable(TableName tableName, int regionNum)
544 throws IOException {
545 int expectedRegions = regionNum;
546 byte[][] splitKeys = new byte[expectedRegions - 1][];
547 for (int i = 1; i < expectedRegions; i++) {
548 byte splitKey = (byte) i;
549 splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
550 }
551
552 HTableDescriptor desc = new HTableDescriptor(tableName);
553 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
554 admin.createTable(desc, splitKeys);
555
556 HTable ht = (HTable) CONNECTION.getTable(tableName);
557 @SuppressWarnings("deprecation")
558 Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
559 assertEquals("Tried to create " + expectedRegions + " regions "
560 + "but only found " + regions.size(), expectedRegions, regions.size());
561 ht.close();
562 }
563 }