View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.net.InetSocketAddress;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.HashMap;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.Random;
35  import java.util.concurrent.atomic.AtomicInteger;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.apache.hadoop.hbase.MiniHBaseCluster;
47  import org.apache.hadoop.hbase.NamespaceDescriptor;
48  import org.apache.hadoop.hbase.ServerName;
49  import org.apache.hadoop.hbase.TableName;
50  import org.apache.hadoop.hbase.client.Admin;
51  import org.apache.hadoop.hbase.client.Connection;
52  import org.apache.hadoop.hbase.client.ConnectionFactory;
53  import org.apache.hadoop.hbase.client.HBaseAdmin;
54  import org.apache.hadoop.hbase.client.HTable;
55  import org.apache.hadoop.hbase.client.MetaScanner;
56  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
57  import org.apache.hadoop.hbase.client.Result;
58  import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
59  import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
60  import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan;
61  import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
62  import org.apache.hadoop.hbase.regionserver.HRegion;
63  import org.apache.hadoop.hbase.regionserver.HRegionServer;
64  import org.apache.hadoop.hbase.util.Bytes;
65  import org.apache.hadoop.hbase.util.Pair;
66  import org.apache.zookeeper.KeeperException;
67  import org.junit.AfterClass;
68  import org.junit.BeforeClass;
69  import org.junit.Test;
70  import org.junit.experimental.categories.Category;
71  
72  
73  @Category(MediumTests.class)
74  public class TestRegionPlacement {
75    final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
76    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
77    private final static int SLAVES = 10;
78    private static Connection CONNECTION;
79    private static Admin admin;
80    private static RegionPlacementMaintainer rp;
81    private static Position[] positions = Position.values();
82    private int lastRegionOnPrimaryRSCount = 0;
83    private int REGION_NUM = 10;
84    private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
85        new HashMap<HRegionInfo, ServerName[]>();
86  
87    @BeforeClass
88    public static void setupBeforeClass() throws Exception {
89      Configuration conf = TEST_UTIL.getConfiguration();
90      // Enable the favored nodes based load balancer
91      conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
92          FavoredNodeLoadBalancer.class, LoadBalancer.class);
93      conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
94      TEST_UTIL.startMiniCluster(SLAVES);
95      CONNECTION = TEST_UTIL.getConnection();
96      admin = CONNECTION.getAdmin();
97      rp = new RegionPlacementMaintainer(conf);
98    }
99  
100   @AfterClass
101   public static void tearDownAfterClass() throws Exception {
102     TEST_UTIL.shutdownMiniCluster();
103   }
104 
105   @Test
106   public void testRegionPlacement() throws Exception {
107     String tableStr = "testRegionAssignment";
108     TableName table = TableName.valueOf(tableStr);
109     // Create a table with REGION_NUM regions.
110     createTable(table, REGION_NUM);
111 
112     TEST_UTIL.waitTableAvailable(table);
113 
114     // Verify all the user regions are assigned to the primary region server
115     // based on the plan
116     verifyRegionOnPrimaryRS(REGION_NUM);
117 
118     FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
119     // Verify all the region server are update with the latest favored nodes
120     verifyRegionServerUpdated(currentPlan);
121     // Test Case 2: To verify whether the region placement tools can
122     // correctly update the new assignment plan to hbase:meta and Region Server.
123     // The new assignment plan is generated by shuffle the existing assignment
124     // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
125     // Shuffle the plan by switching the secondary region server with
126     // the tertiary.
127 
128     // Shuffle the secondary with tertiary favored nodes
129     FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
130         FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
131     // Let the region placement update the hbase:meta and Region Servers
132     rp.updateAssignmentPlan(shuffledPlan);
133 
134     // Verify the region assignment. There are supposed to no region reassignment
135     // All the regions are still on the primary region server
136     verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
137 
138     // Shuffle the plan by switching the primary with secondary and
139     // verify the region reassignment is consistent with the plan.
140     shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
141         FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
142 
143     // Let the region placement update the hbase:meta and Region Servers
144     rp.updateAssignmentPlan(shuffledPlan);
145 
146     verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
147 
148     // also verify that the AssignmentVerificationReport has the correct information
149     RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
150     // we are interested in only one table (and hence one report)
151     rp.setTargetTableName(new String[]{tableStr});
152     List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
153     AssignmentVerificationReport report = reports.get(0);
154     assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
155     assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
156     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
157     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
158     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
159     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
160     assertTrue(report.getUnassignedRegions().size() == 0);
161 
162     // Check when a RS stops, the regions get assigned to their secondary/tertiary
163     killRandomServerAndVerifyAssignment();
164     
165     // also verify that the AssignmentVerificationReport has the correct information
166     reports = rp.verifyRegionPlacement(false);
167     report = reports.get(0);
168     assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
169     assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
170     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
171     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
172     assertTrue("secondary " +
173     report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
174         + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
175         (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
176         || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
177     assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
178         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
179         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
180     RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
181   }
182 
183   private void killRandomServerAndVerifyAssignment() 
184       throws IOException, InterruptedException, KeeperException {
185     ServerName serverToKill = null;
186     int killIndex = 0;
187     Random random = new Random(System.currentTimeMillis());
188     ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
189     LOG.debug("Server holding meta " + metaServer);
190     boolean isNamespaceServer = false;
191     do {
192       // kill a random non-meta server carrying at least one region
193       killIndex = random.nextInt(SLAVES);
194       serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
195       Collection<HRegion> regs =
196           TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
197       isNamespaceServer = false;
198       for (HRegion r : regs) {
199         if (r.getRegionInfo().getTable().getNamespaceAsString()
200             .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
201           isNamespaceServer = true;
202           break;
203         }
204       }
205     } while (ServerName.isSameHostnameAndPort(metaServer, serverToKill) || isNamespaceServer ||
206         TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
207     LOG.debug("Stopping RS " + serverToKill);
208     Map<HRegionInfo, Pair<ServerName, ServerName>> regionsToVerify =
209         new HashMap<HRegionInfo, Pair<ServerName, ServerName>>();
210     // mark the regions to track
211     for (Map.Entry<HRegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
212       ServerName s = entry.getValue()[0];
213       if (ServerName.isSameHostnameAndPort(s, serverToKill)) {
214         regionsToVerify.put(entry.getKey(), new Pair<ServerName, ServerName>(
215             entry.getValue()[1], entry.getValue()[2]));
216         LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
217             entry.getValue()[1] + " " + entry.getValue()[2]);
218       }
219     }
220     int orig = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
221     TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
222     TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
223     int curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
224     while (curr - orig < regionsToVerify.size()) {
225       LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
226           " Current #regions " + curr + " Original #regions " + orig);
227       Thread.sleep(200);
228       curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
229     }
230     // now verify
231     for (Map.Entry<HRegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
232       ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
233           .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
234       Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
235       LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
236           " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
237           + "/" + secondaryTertiaryServers.getSecond());
238       if (!(ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getFirst())||
239           ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getSecond()))){
240         fail("Region " + entry.getKey() + " not present on any of the expected servers");
241       }
242     }
243     // start(reinstate) region server since we killed one before
244     TEST_UTIL.getHBaseCluster().startRegionServer();
245   }
246 
247   /**
248    * Used to test the correctness of this class.
249    */
250   @Test
251   public void testRandomizedMatrix() {
252     int rows = 100;
253     int cols = 100;
254     float[][] matrix = new float[rows][cols];
255     Random random = new Random();
256     for (int i = 0; i < rows; i++) {
257       for (int j = 0; j < cols; j++) {
258         matrix[i][j] = random.nextFloat();
259       }
260     }
261 
262     // Test that inverting a transformed matrix gives the original matrix.
263     RegionPlacementMaintainer.RandomizedMatrix rm =
264       new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
265     float[][] transformed = rm.transform(matrix);
266     float[][] invertedTransformed = rm.invert(transformed);
267     for (int i = 0; i < rows; i++) {
268       for (int j = 0; j < cols; j++) {
269         if (matrix[i][j] != invertedTransformed[i][j]) {
270           throw new RuntimeException();
271         }
272       }
273     }
274 
275     // Test that the indices on a transformed matrix can be inverted to give
276     // the same values on the original matrix.
277     int[] transformedIndices = new int[rows];
278     for (int i = 0; i < rows; i++) {
279       transformedIndices[i] = random.nextInt(cols);
280     }
281     int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
282     float[] transformedValues = new float[rows];
283     float[] invertedTransformedValues = new float[rows];
284     for (int i = 0; i < rows; i++) {
285       transformedValues[i] = transformed[i][transformedIndices[i]];
286       invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
287     }
288     Arrays.sort(transformedValues);
289     Arrays.sort(invertedTransformedValues);
290     if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
291       throw new RuntimeException();
292     }
293   }
294 
295   /**
296    * Shuffle the assignment plan by switching two favored node positions.
297    * @param plan The assignment plan
298    * @param p1 The first switch position
299    * @param p2 The second switch position
300    * @return
301    */
302   private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
303       FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) {
304     FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
305 
306     for (Map.Entry<HRegionInfo, List<ServerName>> entry :
307       plan.getAssignmentMap().entrySet()) {
308       HRegionInfo region = entry.getKey();
309 
310       // copy the server list from the original plan
311       List<ServerName> shuffledServerList = new ArrayList<ServerName>();
312       shuffledServerList.addAll(entry.getValue());
313 
314       // start to shuffle
315       shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
316       shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
317 
318       // update the plan
319       shuffledPlan.updateAssignmentPlan(region, shuffledServerList);
320     }
321     return shuffledPlan;
322   }
323 
324   /**
325    * To verify the region assignment status.
326    * It will check the assignment plan consistency between hbase:meta and
327    * region servers.
328    * Also it will verify weather the number of region movement and
329    * the number regions on the primary region server are expected
330    *
331    * @param plan
332    * @param regionMovementNum
333    * @param numRegionsOnPrimaryRS
334    * @throws InterruptedException
335    * @throws IOException
336    */
337   private void verifyRegionAssignment(FavoredNodesPlan plan,
338       int regionMovementNum, int numRegionsOnPrimaryRS)
339   throws InterruptedException, IOException {
340     // Verify the assignment plan in hbase:meta is consistent with the expected plan.
341     verifyMETAUpdated(plan);
342 
343     // Verify the number of region movement is expected
344     verifyRegionMovementNum(regionMovementNum);
345 
346     // Verify the number of regions is assigned to the primary region server
347     // based on the plan is expected
348     verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
349 
350     // Verify all the online region server are updated with the assignment plan
351     verifyRegionServerUpdated(plan);
352   }
353 
354   /**
355    * Verify the meta has updated to the latest assignment plan
356    * @param plan
357    * @throws IOException
358    */
359   private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
360   throws IOException {
361     FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
362     assertTrue("The assignment plan is NOT consistent with the expected plan ",
363         planFromMETA.equals(expectedPlan));
364   }
365 
366   /**
367    * Verify the number of region movement is expected
368    */
369   private void verifyRegionMovementNum(int expected)
370       throws InterruptedException, IOException {
371     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
372     HMaster m = cluster.getMaster();
373     int lastRegionOpenedCount = m.assignmentManager.getNumRegionsOpened();
374     // get the assignments start to execute
375     m.balance();
376 
377     int retry = 10;
378     long sleep = 3000;
379     int attempt = 0;
380     int currentRegionOpened, regionMovement;
381     do {
382       currentRegionOpened = m.assignmentManager.getNumRegionsOpened();
383       regionMovement= currentRegionOpened - lastRegionOpenedCount;
384       LOG.debug("There are " + regionMovement + "/" + expected +
385           " regions moved after " + attempt + " attempts");
386       Thread.sleep((++attempt) * sleep);
387     } while (regionMovement != expected && attempt <= retry);
388 
389     // update the lastRegionOpenedCount
390     lastRegionOpenedCount = currentRegionOpened;
391 
392     assertEquals("There are only " + regionMovement + " instead of "
393           + expected + " region movement for " + attempt + " attempts",
394           regionMovement, expected);
395   }
396 
397   /**
398    * Verify the number of user regions is assigned to the primary
399    * region server based on the plan is expected
400    * @param expectedNum.
401    * @throws IOException
402    */
403   private void verifyRegionOnPrimaryRS(int expectedNum)
404       throws IOException {
405     lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
406     assertEquals("Only " +  expectedNum + " of user regions running " +
407         "on the primary region server", expectedNum ,
408         lastRegionOnPrimaryRSCount);
409   }
410 
411   /**
412    * Verify all the online region servers has been updated to the
413    * latest assignment plan
414    * @param plan
415    * @throws IOException
416    */
417   private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
418     // Verify all region servers contain the correct favored nodes information
419     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
420     for (int i = 0; i < SLAVES; i++) {
421       HRegionServer rs = cluster.getRegionServer(i);
422       for (HRegion region: rs.getOnlineRegions(
423           TableName.valueOf("testRegionAssignment"))) {
424         InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
425             region.getRegionInfo().getEncodedName());
426         List<ServerName> favoredServerList = plan.getAssignmentMap().get(region.getRegionInfo());
427 
428         // All regions are supposed to have favored nodes,
429         // except for hbase:meta and ROOT
430         if (favoredServerList == null) {
431           HTableDescriptor desc = region.getTableDesc();
432           // Verify they are ROOT and hbase:meta regions since no favored nodes
433           assertNull(favoredSocketAddress);
434           assertTrue("User region " +
435               region.getTableDesc().getTableName() +
436               " should have favored nodes",
437               (desc.isRootRegion() || desc.isMetaRegion()));
438         } else {
439           // For user region, the favored nodes in the region server should be
440           // identical to favored nodes in the assignmentPlan
441           assertTrue(favoredSocketAddress.length == favoredServerList.size());
442           assertTrue(favoredServerList.size() > 0);
443           for (int j = 0; j < favoredServerList.size(); j++) {
444             InetSocketAddress addrFromRS = favoredSocketAddress[j];
445             InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
446                 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
447 
448             assertNotNull(addrFromRS);
449             assertNotNull(addrFromPlan);
450             assertTrue("Region server " + rs.getServerName().getHostAndPort()
451                 + " has the " + positions[j] +
452                 " for region " + region.getRegionNameAsString() + " is " +
453                 addrFromRS + " which is inconsistent with the plan "
454                 + addrFromPlan, addrFromRS.equals(addrFromPlan));
455           }
456         }
457       }
458     }
459   }
460 
461   /**
462    * Check whether regions are assigned to servers consistent with the explicit
463    * hints that are persisted in the hbase:meta table.
464    * Also keep track of the number of the regions are assigned to the
465    * primary region server.
466    * @return the number of regions are assigned to the primary region server
467    * @throws IOException
468    */
469   private int getNumRegionisOnPrimaryRS() throws IOException {
470     final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
471     final AtomicInteger totalRegionNum = new AtomicInteger(0);
472     LOG.info("The start of region placement verification");
473     MetaScannerVisitor visitor = new MetaScannerVisitor() {
474       public boolean processRow(Result result) throws IOException {
475         try {
476           @SuppressWarnings("deprecation")
477           HRegionInfo info = MetaScanner.getHRegionInfo(result);
478           if(info.getTable().getNamespaceAsString()
479               .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
480             return true;
481           }
482           byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
483               HConstants.SERVER_QUALIFIER);
484           byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
485               FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
486           // Add the favored nodes into assignment plan
487           ServerName[] favoredServerList =
488               FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
489           favoredNodesAssignmentPlan.put(info, favoredServerList);
490 
491           Position[] positions = Position.values();
492           if (info != null) {
493             totalRegionNum.incrementAndGet();
494             if (server != null) {
495               ServerName serverName =
496                   ServerName.valueOf(Bytes.toString(server), -1);
497               if (favoredNodes != null) {
498                 String placement = "[NOT FAVORED NODE]";
499                 for (int i = 0; i < favoredServerList.length; i++) {
500                   if (favoredServerList[i].equals(serverName)) {
501                     placement = positions[i].toString();
502                     if (i == Position.PRIMARY.ordinal()) {
503                       regionOnPrimaryNum.incrementAndGet();
504                     }
505                     break;
506                   }
507                 }
508                 LOG.info(info.getRegionNameAsString() + " on " +
509                     serverName + " " + placement);
510               } else {
511                 LOG.info(info.getRegionNameAsString() + " running on " +
512                     serverName + " but there is no favored region server");
513               }
514             } else {
515               LOG.info(info.getRegionNameAsString() +
516                   " not assigned to any server");
517             }
518           }
519           return true;
520         } catch (RuntimeException e) {
521           LOG.error("Result=" + result);
522           throw e;
523         }
524       }
525 
526       @Override
527       public void close() throws IOException {}
528     };
529     MetaScanner.metaScan(CONNECTION, visitor);
530     LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
531         totalRegionNum.intValue() + " regions running on the primary" +
532         " region servers" );
533     return regionOnPrimaryNum.intValue() ;
534   }
535 
536   /**
537    * Create a table with specified table name and region number.
538    * @param tablename
539    * @param regionNum
540    * @return
541    * @throws IOException
542    */
543   private static void createTable(TableName tableName, int regionNum)
544       throws IOException {
545     int expectedRegions = regionNum;
546     byte[][] splitKeys = new byte[expectedRegions - 1][];
547     for (int i = 1; i < expectedRegions; i++) {
548       byte splitKey = (byte) i;
549       splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
550     }
551 
552     HTableDescriptor desc = new HTableDescriptor(tableName);
553     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
554     admin.createTable(desc, splitKeys);
555 
556     HTable ht = (HTable) CONNECTION.getTable(tableName);
557     @SuppressWarnings("deprecation")
558     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
559     assertEquals("Tried to create " + expectedRegions + " regions "
560         + "but only found " + regions.size(), expectedRegions, regions.size());
561     ht.close();
562   }
563 }