View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.net.InetSocketAddress;
27  import java.util.ArrayList;
28  import java.util.Collection;
29  import java.util.HashMap;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.Set;
33  import java.util.concurrent.atomic.AtomicInteger;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.HBaseIOException;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.MediumTests;
46  import org.apache.hadoop.hbase.MiniHBaseCluster;
47  import org.apache.hadoop.hbase.NamespaceDescriptor;
48  import org.apache.hadoop.hbase.ServerName;
49  import org.apache.hadoop.hbase.client.HBaseAdmin;
50  import org.apache.hadoop.hbase.client.HTable;
51  import org.apache.hadoop.hbase.client.MetaScanner;
52  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
53  import org.apache.hadoop.hbase.client.Result;
54  import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
55  import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
56  import org.apache.hadoop.hbase.master.balancer.FavoredNodes.Position;
57  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
58  import org.apache.hadoop.hbase.regionserver.HRegion;
59  import org.apache.hadoop.hbase.regionserver.HRegionServer;
60  import org.apache.hadoop.hbase.util.Bytes;
61  import org.junit.AfterClass;
62  import org.junit.BeforeClass;
63  import org.junit.Test;
64  import org.junit.experimental.categories.Category;
65  
66  @Category(MediumTests.class)
67  public class TestRegionPlacement {
68    final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
69    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
70    private final static int SLAVES = 10;
71    private static HBaseAdmin admin;
72    private static Position[] positions = Position.values();
73    private int REGION_NUM = 10;
74    private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
75        new HashMap<HRegionInfo, ServerName[]>();
76    private final static int PRIMARY = Position.PRIMARY.ordinal();
77    private final static int SECONDARY = Position.SECONDARY.ordinal();
78    private final static int TERTIARY = Position.TERTIARY.ordinal();
79  
80    @BeforeClass
81    public static void setupBeforeClass() throws Exception {
82      Configuration conf = TEST_UTIL.getConfiguration();
83      // Enable the favored nodes based load balancer
84      conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
85          FavoredNodeLoadBalancer.class, LoadBalancer.class);
86      conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
87      TEST_UTIL.startMiniCluster(SLAVES);
88      admin = new HBaseAdmin(conf);
89    }
90  
91    @AfterClass
92    public static void tearDownAfterClass() throws Exception {
93      TEST_UTIL.shutdownMiniCluster();
94    }
95  
96    @Test
97    public void testFavoredNodesPresentForRoundRobinAssignment() throws HBaseIOException {
98      LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
99      balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
100     List<ServerName> servers = new ArrayList<ServerName>();
101     for (int i = 0; i < SLAVES; i++) {
102       ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
103       servers.add(server);
104     }
105     List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
106     HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
107     regions.add(region);
108     Map<ServerName,List<HRegionInfo>> assignmentMap = balancer.roundRobinAssignment(regions,
109         servers);
110     Set<ServerName> serverBefore = assignmentMap.keySet();
111     List<ServerName> favoredNodesBefore =
112         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
113     assertTrue(favoredNodesBefore.size() == 3);
114     // the primary RS should be the one that the balancer's assignment returns
115     assertTrue(ServerName.isSameHostnameAndPort(serverBefore.iterator().next(),
116         favoredNodesBefore.get(PRIMARY)));
117     // now remove the primary from the list of available servers
118     List<ServerName> removedServers = removeMatchingServers(serverBefore, servers);
119     // call roundRobinAssignment with the modified servers list
120     assignmentMap = balancer.roundRobinAssignment(regions, servers);
121     List<ServerName> favoredNodesAfter =
122         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
123     assertTrue(favoredNodesAfter.size() == 3);
124     // We don't expect the favored nodes assignments to change in multiple calls
125     // to the roundRobinAssignment method in the balancer (relevant for AssignmentManager.assign
126     // failures)
127     assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
128     Set<ServerName> serverAfter = assignmentMap.keySet();
129     // We expect the new RegionServer assignee to be one of the favored nodes
130     // chosen earlier.
131     assertTrue(ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
132                  favoredNodesBefore.get(SECONDARY)) ||
133                ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
134                  favoredNodesBefore.get(TERTIARY)));
135 
136     // put back the primary in the list of available servers
137     servers.addAll(removedServers);
138     // now roundRobinAssignment with the modified servers list should return the primary
139     // as the regionserver assignee
140     assignmentMap = balancer.roundRobinAssignment(regions, servers);
141     Set<ServerName> serverWithPrimary = assignmentMap.keySet();
142     assertTrue(serverBefore.containsAll(serverWithPrimary));
143 
144     // Make all the favored nodes unavailable for assignment
145     removeMatchingServers(favoredNodesAfter, servers);
146     // call roundRobinAssignment with the modified servers list
147     assignmentMap = balancer.roundRobinAssignment(regions, servers);
148     List<ServerName> favoredNodesNow =
149         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
150     assertTrue(favoredNodesNow.size() == 3);
151     assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
152         !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
153         !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
154   }
155 
156   @Test
157   public void testFavoredNodesPresentForRandomAssignment() throws HBaseIOException {
158     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
159     balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
160     List<ServerName> servers = new ArrayList<ServerName>();
161     for (int i = 0; i < SLAVES; i++) {
162       ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
163       servers.add(server);
164     }
165     List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
166     HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
167     regions.add(region);
168     ServerName serverBefore = balancer.randomAssignment(region, servers);
169     List<ServerName> favoredNodesBefore =
170         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
171     assertTrue(favoredNodesBefore.size() == 3);
172     // the primary RS should be the one that the balancer's assignment returns
173     assertTrue(ServerName.isSameHostnameAndPort(serverBefore,favoredNodesBefore.get(PRIMARY)));
174     // now remove the primary from the list of servers
175     removeMatchingServers(serverBefore, servers);
176     // call randomAssignment with the modified servers list
177     ServerName serverAfter = balancer.randomAssignment(region, servers);
178     List<ServerName> favoredNodesAfter =
179         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
180     assertTrue(favoredNodesAfter.size() == 3);
181     // We don't expect the favored nodes assignments to change in multiple calls
182     // to the randomAssignment method in the balancer (relevant for AssignmentManager.assign
183     // failures)
184     assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
185     // We expect the new RegionServer assignee to be one of the favored nodes
186     // chosen earlier.
187     assertTrue(ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(SECONDARY)) ||
188                ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(TERTIARY)));
189     // Make all the favored nodes unavailable for assignment
190     removeMatchingServers(favoredNodesAfter, servers);
191     // call randomAssignment with the modified servers list
192     balancer.randomAssignment(region, servers);
193     List<ServerName> favoredNodesNow =
194         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
195     assertTrue(favoredNodesNow.size() == 3);
196     assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
197         !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
198         !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
199   }
200 
201   @Test(timeout = 180000)
202   public void testRegionPlacement() throws Exception {
203     // Create a table with REGION_NUM regions.
204     createTable("testRegionAssignment", REGION_NUM);
205 
206     TEST_UTIL.waitTableAvailable(Bytes.toBytes("testRegionAssignment"));
207 
208     // Verify all the user regions are assigned to the primary region server
209     // based on the plan
210     countRegionOnPrimaryRS(REGION_NUM);
211 
212     // Verify all the region server are update with the latest favored nodes
213     verifyRegionServerUpdated();
214   }
215 
216   private List<ServerName> removeMatchingServers(ServerName serverWithoutStartCode,
217       List<ServerName> servers) {
218     List<ServerName> serversToRemove = new ArrayList<ServerName>();
219     for (ServerName s : servers) {
220       if (ServerName.isSameHostnameAndPort(s, serverWithoutStartCode)) {
221         serversToRemove.add(s);
222       }
223     }
224     servers.removeAll(serversToRemove);
225     return serversToRemove;
226   }
227 
228   private List<ServerName> removeMatchingServers(Collection<ServerName> serversWithoutStartCode,
229       List<ServerName> servers) {
230     List<ServerName> serversToRemove = new ArrayList<ServerName>();
231     for (ServerName s : serversWithoutStartCode) {
232       serversToRemove.addAll(removeMatchingServers(s, servers));
233     }
234     return serversToRemove;
235   }
236 
237   /**
238    * Verify the number of user regions is assigned to the primary
239    * region server based on the plan is expected
240    * @param expectedNum.
241    * @throws IOException
242    */
243   private void countRegionOnPrimaryRS(int expectedNum)
244       throws IOException {
245     int lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
246     assertEquals("Only " +  expectedNum + " of user regions running " +
247         "on the primary region server", expectedNum ,
248         lastRegionOnPrimaryRSCount);
249   }
250 
251   /**
252    * Verify all the online region servers has been updated to the
253    * latest assignment plan
254    * @param plan
255    * @throws IOException
256    */
257   private void verifyRegionServerUpdated() throws IOException {
258     // Verify all region servers contain the correct favored nodes information
259     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
260     for (int i = 0; i < SLAVES; i++) {
261       HRegionServer rs = cluster.getRegionServer(i);
262       for (HRegion region: rs.getOnlineRegions(
263           TableName.valueOf("testRegionAssignment"))) {
264         InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
265             region.getRegionInfo().getEncodedName());
266         ServerName[] favoredServerList = favoredNodesAssignmentPlan.get(region.getRegionInfo());
267 
268         // All regions are supposed to have favored nodes,
269         // except for META and ROOT
270         if (favoredServerList == null) {
271           HTableDescriptor desc = region.getTableDesc();
272           // Verify they are ROOT and META regions since no favored nodes
273           assertNull(favoredSocketAddress);
274           assertTrue("User region " +
275               region.getTableDesc().getTableName() +
276               " should have favored nodes",
277               (desc.isRootRegion() || desc.isMetaRegion()));
278         } else {
279           // For user region, the favored nodes in the region server should be
280           // identical to favored nodes in the assignmentPlan
281           assertTrue(favoredSocketAddress.length == favoredServerList.length);
282           assertTrue(favoredServerList.length > 0);
283           for (int j = 0; j < favoredServerList.length; j++) {
284             InetSocketAddress addrFromRS = favoredSocketAddress[j];
285             InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
286                 favoredServerList[j].getHostname(), favoredServerList[j].getPort());
287 
288             assertNotNull(addrFromRS);
289             assertNotNull(addrFromPlan);
290             assertTrue("Region server " + rs.getServerName().getHostAndPort()
291                 + " has the " + positions[j] +
292                 " for region " + region.getRegionNameAsString() + " is " +
293                 addrFromRS + " which is inconsistent with the plan "
294                 + addrFromPlan, addrFromRS.equals(addrFromPlan));
295           }
296         }
297       }
298     }
299   }
300 
301   /**
302    * Check whether regions are assigned to servers consistent with the explicit
303    * hints that are persisted in the META table.
304    * Also keep track of the number of the regions are assigned to the
305    * primary region server.
306    * @return the number of regions are assigned to the primary region server
307    * @throws IOException
308    */
309   private int getNumRegionisOnPrimaryRS() throws IOException {
310     final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
311     final AtomicInteger totalRegionNum = new AtomicInteger(0);
312     LOG.info("The start of region placement verification");
313     MetaScannerVisitor visitor = new MetaScannerVisitor() {
314       public boolean processRow(Result result) throws IOException {
315         try {
316           HRegionInfo info = MetaScanner.getHRegionInfo(result);
317           if(info.getTableName().getNamespaceAsString()
318               .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
319             return true;
320           }
321           byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
322               HConstants.SERVER_QUALIFIER);
323           byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
324               FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
325           // Add the favored nodes into assignment plan
326           ServerName[] favoredServerList =
327               FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
328           favoredNodesAssignmentPlan.put(info, favoredServerList);
329 
330           Position[] positions = Position.values();
331           if (info != null) {
332             totalRegionNum.incrementAndGet();
333             if (server != null) {
334               ServerName serverName =
335                   new ServerName(Bytes.toString(server), -1);
336               if (favoredNodes != null) {
337                 String placement = "[NOT FAVORED NODE]";
338                 for (int i = 0; i < favoredServerList.length; i++) {
339                   if (favoredServerList[i].equals(serverName)) {
340                     placement = positions[i].toString();
341                     if (i == Position.PRIMARY.ordinal()) {
342                       regionOnPrimaryNum.incrementAndGet();
343                     }
344                     break;
345                   }
346                 }
347                 LOG.info(info.getRegionNameAsString() + " on " +
348                     serverName + " " + placement);
349               } else {
350                 LOG.info(info.getRegionNameAsString() + " running on " +
351                     serverName + " but there is no favored region server");
352               }
353             } else {
354               LOG.info(info.getRegionNameAsString() +
355                   " not assigned to any server");
356             }
357           }
358           return true;
359         } catch (RuntimeException e) {
360           LOG.error("Result=" + result);
361           throw e;
362         }
363       }
364 
365       @Override
366       public void close() throws IOException {}
367     };
368     MetaScanner.metaScan(TEST_UTIL.getConfiguration(), visitor);
369     LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
370         totalRegionNum.intValue() + " regions running on the primary" +
371         " region servers" );
372     return regionOnPrimaryNum.intValue() ;
373   }
374 
375   /**
376    * Create a table with specified table name and region number.
377    * @param table
378    * @param regionNum
379    * @return
380    * @throws IOException
381    */
382   private static void createTable(String table, int regionNum)
383       throws IOException {
384     byte[] tableName = Bytes.toBytes(table);
385     int expectedRegions = regionNum;
386     byte[][] splitKeys = new byte[expectedRegions - 1][];
387     for (int i = 1; i < expectedRegions; i++) {
388       byte splitKey = (byte) i;
389       splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
390     }
391 
392     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
393     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
394     admin.createTable(desc, splitKeys);
395 
396     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
397     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
398     assertEquals("Tried to create " + expectedRegions + " regions "
399         + "but only found " + regions.size(), expectedRegions, regions.size());
400   }
401 }