1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
25 import org.apache.hadoop.hbase.executor.EventType;
26 import org.apache.hadoop.hbase.executor.ExecutorService;
27 import org.apache.hadoop.hbase.executor.ExecutorType;
28 import org.apache.hadoop.hbase.master.AssignmentManager;
29 import org.apache.hadoop.hbase.master.HMaster;
30 import org.apache.hadoop.hbase.master.LoadBalancer;
31 import org.apache.hadoop.hbase.master.RegionPlan;
32 import org.apache.hadoop.hbase.master.RegionState;
33 import org.apache.hadoop.hbase.master.ServerManager;
34 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
35 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
36 import org.apache.hadoop.hbase.testclassification.MediumTests;
37 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
38 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
39 import org.junit.AfterClass;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43 import org.mockito.Mockito;
44
45 import java.util.ArrayList;
46 import java.util.HashMap;
47 import java.util.HashSet;
48 import java.util.List;
49 import java.util.Map;
50 import java.util.Map.Entry;
51 import java.util.Set;
52
53 import static org.junit.Assert.assertFalse;
54 import static org.junit.Assert.assertNotEquals;
55 import static org.junit.Assert.assertTrue;
56
57
58
59
60
61 @Category(MediumTests.class)
62 public class TestDrainingServer {
63 private static final Log LOG = LogFactory.getLog(TestDrainingServer.class);
64 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65 private Abortable abortable = new Abortable() {
66 @Override
67 public boolean isAborted() {
68 return false;
69 }
70
71 @Override
72 public void abort(String why, Throwable e) {
73 }
74 };
75
76 @AfterClass
77 public static void afterClass() throws Exception {
78 TEST_UTIL.shutdownMiniZKCluster();
79 }
80
81 @BeforeClass
82 public static void beforeClass() throws Exception {
83 TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
84 TEST_UTIL.startMiniZKCluster();
85 }
86
87 @Test
88 public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
89 AssignmentManager am;
90 Configuration conf = TEST_UTIL.getConfiguration();
91 final HMaster master = Mockito.mock(HMaster.class);
92 final Server server = Mockito.mock(Server.class);
93 final ServerManager serverManager = Mockito.mock(ServerManager.class);
94 final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
95 final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
96 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
97 final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("table_test"),
98 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
99
100 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
101 "zkWatcher-Test", abortable, true);
102
103 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
104
105 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
106 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
107
108 Mockito.when(server.getConfiguration()).thenReturn(conf);
109 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
110 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
111
112 CoordinatedStateManager cp = new ZkCoordinatedStateManager();
113 cp.initialize(server);
114 cp.start();
115
116 Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
117
118 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
119 Mockito.when(serverManager.getOnlineServersList())
120 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
121
122 Mockito.when(serverManager.createDestinationServersList())
123 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
124 Mockito.when(serverManager.createDestinationServersList(null))
125 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
126
127 for (ServerName sn : onlineServers.keySet()) {
128 Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
129 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
130 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
131 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
132 .thenReturn(RegionOpeningState.OPENED);
133 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
134 .thenReturn(RegionOpeningState.OPENED);
135 Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
136 }
137
138 Mockito.when(master.getServerManager()).thenReturn(serverManager);
139
140 am = new AssignmentManager(server, serverManager,
141 balancer, startupMasterExecutor("mockExecutorService"), null, null);
142
143 Mockito.when(master.getAssignmentManager()).thenReturn(am);
144 Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);
145
146 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
147
148 zkWatcher.registerListenerFirst(am);
149
150 addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);
151
152 am.assign(REGIONINFO, true);
153
154 setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
155 setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);
156
157 am.waitForAssignment(REGIONINFO);
158
159 assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
160 assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
161 }
162
163 @Test
164 public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
165 Configuration conf = TEST_UTIL.getConfiguration();
166 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
167 AssignmentManager am;
168 final HMaster master = Mockito.mock(HMaster.class);
169 final Server server = Mockito.mock(Server.class);
170 final ServerManager serverManager = Mockito.mock(ServerManager.class);
171 final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
172 final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
173 final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
174 final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
175 final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
176 final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();
177
178 Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();
179
180 HRegionInfo REGIONINFO_A = new HRegionInfo(TableName.valueOf("table_A"),
181 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
182 HRegionInfo REGIONINFO_B = new HRegionInfo(TableName.valueOf("table_B"),
183 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
184 HRegionInfo REGIONINFO_C = new HRegionInfo(TableName.valueOf("table_C"),
185 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
186 HRegionInfo REGIONINFO_D = new HRegionInfo(TableName.valueOf("table_D"),
187 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
188 HRegionInfo REGIONINFO_E = new HRegionInfo(TableName.valueOf("table_E"),
189 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
190
191 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
192 List<ServerName> drainedServers = new ArrayList<ServerName>();
193
194 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
195 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
196 onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
197 onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
198 onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);
199
200 bulk.put(REGIONINFO_A, SERVERNAME_A);
201 bulk.put(REGIONINFO_B, SERVERNAME_B);
202 bulk.put(REGIONINFO_C, SERVERNAME_C);
203 bulk.put(REGIONINFO_D, SERVERNAME_D);
204 bulk.put(REGIONINFO_E, SERVERNAME_E);
205
206 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
207 "zkWatcher-BulkAssignTest", abortable, true);
208
209 Mockito.when(server.getConfiguration()).thenReturn(conf);
210 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
211 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
212
213 CoordinatedStateManager cp = new ZkCoordinatedStateManager();
214 cp.initialize(server);
215 cp.start();
216
217 Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
218
219 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
220 Mockito.when(serverManager.getOnlineServersList()).thenReturn(
221 new ArrayList<ServerName>(onlineServers.keySet()));
222
223 Mockito.when(serverManager.createDestinationServersList()).thenReturn(
224 new ArrayList<ServerName>(onlineServers.keySet()));
225 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(
226 new ArrayList<ServerName>(onlineServers.keySet()));
227
228 for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
229 Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
230 Mockito.when(serverManager.sendRegionClose(entry.getValue(),
231 entry.getKey(), -1)).thenReturn(true);
232 Mockito.when(serverManager.sendRegionOpen(entry.getValue(),
233 entry.getKey(), -1, null)).thenReturn(RegionOpeningState.OPENED);
234 Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
235 }
236
237 Mockito.when(master.getServerManager()).thenReturn(serverManager);
238
239 drainedServers.add(SERVERNAME_A);
240 drainedServers.add(SERVERNAME_B);
241 drainedServers.add(SERVERNAME_C);
242 drainedServers.add(SERVERNAME_D);
243
244 am = new AssignmentManager(server, serverManager,
245 balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null);
246
247 Mockito.when(master.getAssignmentManager()).thenReturn(am);
248
249 zkWatcher.registerListener(am);
250
251 for (ServerName drained : drainedServers) {
252 addServerToDrainedList(drained, onlineServers, serverManager);
253 }
254
255 am.assign(bulk);
256
257 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
258 for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
259 setRegionOpenedOnZK(zkWatcher, entry.getValue().getServerName(),
260 entry.getValue().getRegion());
261 }
262
263 am.waitForAssignment(REGIONINFO_A);
264 am.waitForAssignment(REGIONINFO_B);
265 am.waitForAssignment(REGIONINFO_C);
266 am.waitForAssignment(REGIONINFO_D);
267 am.waitForAssignment(REGIONINFO_E);
268
269 Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
270 for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
271 LOG.info("Region Assignment: "
272 + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue());
273 bunchServersAssigned.add(entry.getValue());
274 }
275
276 for (ServerName sn : drainedServers) {
277 assertFalse(bunchServersAssigned.contains(sn));
278 }
279 }
280
281 private void addServerToDrainedList(ServerName serverName,
282 Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) {
283 onlineServers.remove(serverName);
284 List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet());
285 Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers);
286 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers);
287 }
288
289 private void setRegionOpenedOnZK(final ZooKeeperWatcher zkWatcher, final ServerName serverName,
290 HRegionInfo hregionInfo) throws Exception {
291 int version = ZKAssign.getVersion(zkWatcher, hregionInfo);
292 int versionTransition = ZKAssign.transitionNode(zkWatcher,
293 hregionInfo, serverName, EventType.M_ZK_REGION_OFFLINE,
294 EventType.RS_ZK_REGION_OPENING, version);
295 ZKAssign.transitionNodeOpened(zkWatcher, hregionInfo, serverName, versionTransition);
296 }
297
298 private ExecutorService startupMasterExecutor(final String name) {
299 ExecutorService executor = new ExecutorService(name);
300 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
301 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
302 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
303 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
304 return executor;
305 }
306 }