View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
25  import org.apache.hadoop.hbase.executor.EventType;
26  import org.apache.hadoop.hbase.executor.ExecutorService;
27  import org.apache.hadoop.hbase.executor.ExecutorType;
28  import org.apache.hadoop.hbase.master.AssignmentManager;
29  import org.apache.hadoop.hbase.master.HMaster;
30  import org.apache.hadoop.hbase.master.LoadBalancer;
31  import org.apache.hadoop.hbase.master.RegionPlan;
32  import org.apache.hadoop.hbase.master.RegionState;
33  import org.apache.hadoop.hbase.master.ServerManager;
34  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
35  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
36  import org.apache.hadoop.hbase.testclassification.MediumTests;
37  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
38  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
39  import org.junit.AfterClass;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  import org.mockito.Mockito;
44  
45  import java.util.ArrayList;
46  import java.util.HashMap;
47  import java.util.HashSet;
48  import java.util.List;
49  import java.util.Map;
50  import java.util.Map.Entry;
51  import java.util.Set;
52  
53  import static org.junit.Assert.assertFalse;
54  import static org.junit.Assert.assertNotEquals;
55  import static org.junit.Assert.assertTrue;
56  
57  
58  /**
59   * Test the draining servers feature.
60   */
61  @Category(MediumTests.class)
62  public class TestDrainingServer {
63    private static final Log LOG = LogFactory.getLog(TestDrainingServer.class);
64    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65    private Abortable abortable = new Abortable() {
66      @Override
67      public boolean isAborted() {
68        return false;
69      }
70  
71      @Override
72      public void abort(String why, Throwable e) {
73      }
74    };
75  
76    @AfterClass
77    public static void afterClass() throws Exception {
78      TEST_UTIL.shutdownMiniZKCluster();
79    }
80    
81    @BeforeClass
82    public static void beforeClass() throws Exception {
83      TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
84      TEST_UTIL.startMiniZKCluster();
85    }
86  
87    @Test
88    public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
89      AssignmentManager am;
90      Configuration conf = TEST_UTIL.getConfiguration();
91      final HMaster master = Mockito.mock(HMaster.class);
92      final Server server = Mockito.mock(Server.class);
93      final ServerManager serverManager = Mockito.mock(ServerManager.class);
94      final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
95      final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
96      LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
97      final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("table_test"),
98          HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
99  
100     ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
101       "zkWatcher-Test", abortable, true);
102 
103     Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
104 
105     onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
106     onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
107 
108     Mockito.when(server.getConfiguration()).thenReturn(conf);
109     Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
110     Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
111 
112     CoordinatedStateManager cp = new ZkCoordinatedStateManager();
113     cp.initialize(server);
114     cp.start();
115 
116     Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
117 
118     Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
119     Mockito.when(serverManager.getOnlineServersList())
120     .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
121     
122     Mockito.when(serverManager.createDestinationServersList())
123         .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
124     Mockito.when(serverManager.createDestinationServersList(null))
125         .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
126     
127     for (ServerName sn : onlineServers.keySet()) {
128       Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
129       Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
130       Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
131       Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
132       .thenReturn(RegionOpeningState.OPENED);
133       Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
134       .thenReturn(RegionOpeningState.OPENED);
135       Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
136     }
137 
138     Mockito.when(master.getServerManager()).thenReturn(serverManager);
139 
140     am = new AssignmentManager(server, serverManager,
141         balancer, startupMasterExecutor("mockExecutorService"), null, null);
142 
143     Mockito.when(master.getAssignmentManager()).thenReturn(am);
144     Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);
145 
146     am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
147 
148     zkWatcher.registerListenerFirst(am);
149 
150     addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);
151 
152     am.assign(REGIONINFO, true);
153 
154     setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
155     setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);
156 
157     am.waitForAssignment(REGIONINFO);
158 
159     assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
160     assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
161   }
162 
163   @Test
164   public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
165     Configuration conf = TEST_UTIL.getConfiguration();
166     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
167     AssignmentManager am;
168     final HMaster master = Mockito.mock(HMaster.class);
169     final Server server = Mockito.mock(Server.class);
170     final ServerManager serverManager = Mockito.mock(ServerManager.class);
171     final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
172     final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
173     final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
174     final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
175     final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
176     final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();
177 
178     Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();
179     
180     HRegionInfo REGIONINFO_A = new HRegionInfo(TableName.valueOf("table_A"),
181         HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
182     HRegionInfo REGIONINFO_B = new HRegionInfo(TableName.valueOf("table_B"),
183       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
184     HRegionInfo REGIONINFO_C = new HRegionInfo(TableName.valueOf("table_C"),
185       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
186     HRegionInfo REGIONINFO_D = new HRegionInfo(TableName.valueOf("table_D"),
187       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
188     HRegionInfo REGIONINFO_E = new HRegionInfo(TableName.valueOf("table_E"),
189       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
190 
191     Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
192     List<ServerName> drainedServers = new ArrayList<ServerName>();
193 
194     onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
195     onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
196     onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
197     onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
198     onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);
199 
200     bulk.put(REGIONINFO_A, SERVERNAME_A);
201     bulk.put(REGIONINFO_B, SERVERNAME_B);
202     bulk.put(REGIONINFO_C, SERVERNAME_C);
203     bulk.put(REGIONINFO_D, SERVERNAME_D);
204     bulk.put(REGIONINFO_E, SERVERNAME_E);
205 
206     ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
207         "zkWatcher-BulkAssignTest", abortable, true);
208 
209     Mockito.when(server.getConfiguration()).thenReturn(conf);
210     Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
211     Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
212 
213     CoordinatedStateManager cp = new ZkCoordinatedStateManager();
214     cp.initialize(server);
215     cp.start();
216 
217     Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
218 
219     Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
220     Mockito.when(serverManager.getOnlineServersList()).thenReturn(
221       new ArrayList<ServerName>(onlineServers.keySet()));
222     
223     Mockito.when(serverManager.createDestinationServersList()).thenReturn(
224       new ArrayList<ServerName>(onlineServers.keySet()));
225     Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(
226       new ArrayList<ServerName>(onlineServers.keySet()));
227     
228     for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
229       Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
230       Mockito.when(serverManager.sendRegionClose(entry.getValue(), 
231         entry.getKey(), -1)).thenReturn(true);
232       Mockito.when(serverManager.sendRegionOpen(entry.getValue(), 
233         entry.getKey(), -1, null)).thenReturn(RegionOpeningState.OPENED);  
234       Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
235     }
236     
237     Mockito.when(master.getServerManager()).thenReturn(serverManager);
238 
239     drainedServers.add(SERVERNAME_A);
240     drainedServers.add(SERVERNAME_B);
241     drainedServers.add(SERVERNAME_C);
242     drainedServers.add(SERVERNAME_D);
243 
244     am = new AssignmentManager(server, serverManager,
245       balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null);
246     
247     Mockito.when(master.getAssignmentManager()).thenReturn(am);
248 
249     zkWatcher.registerListener(am);
250     
251     for (ServerName drained : drainedServers) {
252       addServerToDrainedList(drained, onlineServers, serverManager);
253     }
254     
255     am.assign(bulk);
256 
257     Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
258     for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
259       setRegionOpenedOnZK(zkWatcher, entry.getValue().getServerName(), 
260         entry.getValue().getRegion());
261     }
262     
263     am.waitForAssignment(REGIONINFO_A);
264     am.waitForAssignment(REGIONINFO_B);
265     am.waitForAssignment(REGIONINFO_C);
266     am.waitForAssignment(REGIONINFO_D);
267     am.waitForAssignment(REGIONINFO_E);
268     
269     Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
270     for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
271       LOG.info("Region Assignment: " 
272           + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue());
273       bunchServersAssigned.add(entry.getValue());
274     }
275     
276     for (ServerName sn : drainedServers) {
277       assertFalse(bunchServersAssigned.contains(sn));
278     }
279   }
280 
281   private void addServerToDrainedList(ServerName serverName, 
282       Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) {
283     onlineServers.remove(serverName);
284     List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet());
285     Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers);
286     Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers);
287   }
288 
289   private void setRegionOpenedOnZK(final ZooKeeperWatcher zkWatcher, final ServerName serverName,
290                                    HRegionInfo hregionInfo) throws Exception {
291     int version = ZKAssign.getVersion(zkWatcher, hregionInfo);
292     int versionTransition = ZKAssign.transitionNode(zkWatcher,
293         hregionInfo, serverName, EventType.M_ZK_REGION_OFFLINE,
294         EventType.RS_ZK_REGION_OPENING, version);
295     ZKAssign.transitionNodeOpened(zkWatcher, hregionInfo, serverName, versionTransition);
296   }
297 
298   private ExecutorService startupMasterExecutor(final String name) {
299     ExecutorService executor = new ExecutorService(name);
300     executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
301     executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
302     executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
303     executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
304     return executor;
305   }
306 }