1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.catalog.CatalogTracker;
25 import org.apache.hadoop.hbase.executor.EventType;
26 import org.apache.hadoop.hbase.executor.ExecutorService;
27 import org.apache.hadoop.hbase.executor.ExecutorType;
28 import org.apache.hadoop.hbase.master.AssignmentManager;
29 import org.apache.hadoop.hbase.master.HMaster;
30 import org.apache.hadoop.hbase.master.LoadBalancer;
31 import org.apache.hadoop.hbase.master.RegionPlan;
32 import org.apache.hadoop.hbase.master.RegionState;
33 import org.apache.hadoop.hbase.master.ServerManager;
34 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
35 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
36 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
37 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
38 import org.junit.AfterClass;
39 import org.junit.BeforeClass;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42 import org.mockito.Mockito;
43
44 import java.util.ArrayList;
45 import java.util.HashMap;
46 import java.util.HashSet;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.Map.Entry;
50 import java.util.Set;
51
52 import static org.junit.Assert.assertFalse;
53 import static org.junit.Assert.assertNotEquals;
54 import static org.junit.Assert.assertTrue;
55
56
57
58
59
60 @Category(MediumTests.class)
61 public class TestDrainingServer {
62 private static final Log LOG = LogFactory.getLog(TestDrainingServer.class);
63 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
64 private Abortable abortable = new Abortable() {
65 @Override
66 public boolean isAborted() {
67 return false;
68 }
69
70 @Override
71 public void abort(String why, Throwable e) {
72 }
73 };
74
75 @AfterClass
76 public static void afterClass() throws Exception {
77 TEST_UTIL.shutdownMiniZKCluster();
78 }
79
80 @BeforeClass
81 public static void beforeClass() throws Exception {
82 TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
83 TEST_UTIL.startMiniZKCluster();
84 }
85
86 @Test
87 public void testAssignmentManagerDoesntUseDrainingServer() throws Exception {
88 AssignmentManager am;
89 Configuration conf = TEST_UTIL.getConfiguration();
90 final HMaster master = Mockito.mock(HMaster.class);
91 final Server server = Mockito.mock(Server.class);
92 final ServerManager serverManager = Mockito.mock(ServerManager.class);
93 final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000);
94 final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000);
95 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
96 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
97 final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("table_test"),
98 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
99
100 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
101 "zkWatcher-Test", abortable, true);
102
103 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
104
105 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
106 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
107
108 Mockito.when(server.getConfiguration()).thenReturn(conf);
109 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
110 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
111
112 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
113 Mockito.when(serverManager.getOnlineServersList())
114 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
115
116 Mockito.when(serverManager.createDestinationServersList())
117 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
118 Mockito.when(serverManager.createDestinationServersList(null))
119 .thenReturn(new ArrayList<ServerName>(onlineServers.keySet()));
120
121 for (ServerName sn : onlineServers.keySet()) {
122 Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true);
123 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true);
124 Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true);
125 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>()))
126 .thenReturn(RegionOpeningState.OPENED);
127 Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null))
128 .thenReturn(RegionOpeningState.OPENED);
129 Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true);
130 }
131
132 Mockito.when(master.getServerManager()).thenReturn(serverManager);
133
134 am = new AssignmentManager(server, serverManager, catalogTracker,
135 balancer, startupMasterExecutor("mockExecutorService"), null, null);
136
137 Mockito.when(master.getAssignmentManager()).thenReturn(am);
138 Mockito.when(master.getZooKeeperWatcher()).thenReturn(zkWatcher);
139 Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher);
140
141 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A));
142
143 zkWatcher.registerListenerFirst(am);
144
145 addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager);
146
147 am.assign(REGIONINFO, true);
148
149 setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO);
150 setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO);
151
152 am.waitForAssignment(REGIONINFO);
153
154 assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO));
155 assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A);
156 }
157
158 @Test
159 public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception {
160 Configuration conf = TEST_UTIL.getConfiguration();
161 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf);
162 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
163 AssignmentManager am;
164 final HMaster master = Mockito.mock(HMaster.class);
165 final Server server = Mockito.mock(Server.class);
166 final ServerManager serverManager = Mockito.mock(ServerManager.class);
167 final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
168 final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
169 final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
170 final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000);
171 final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000);
172 final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>();
173
174 Set<ServerName> bunchServersAssigned = new HashSet<ServerName>();
175
176 HRegionInfo REGIONINFO_A = new HRegionInfo(TableName.valueOf("table_A"),
177 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
178 HRegionInfo REGIONINFO_B = new HRegionInfo(TableName.valueOf("table_B"),
179 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
180 HRegionInfo REGIONINFO_C = new HRegionInfo(TableName.valueOf("table_C"),
181 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
182 HRegionInfo REGIONINFO_D = new HRegionInfo(TableName.valueOf("table_D"),
183 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
184 HRegionInfo REGIONINFO_E = new HRegionInfo(TableName.valueOf("table_E"),
185 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
186
187 Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
188 List<ServerName> drainedServers = new ArrayList<ServerName>();
189
190 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
191 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
192 onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD);
193 onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD);
194 onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD);
195
196 bulk.put(REGIONINFO_A, SERVERNAME_A);
197 bulk.put(REGIONINFO_B, SERVERNAME_B);
198 bulk.put(REGIONINFO_C, SERVERNAME_C);
199 bulk.put(REGIONINFO_D, SERVERNAME_D);
200 bulk.put(REGIONINFO_E, SERVERNAME_E);
201
202 ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
203 "zkWatcher-BulkAssignTest", abortable, true);
204
205 Mockito.when(server.getConfiguration()).thenReturn(conf);
206 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1"));
207 Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher);
208
209 Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers);
210 Mockito.when(serverManager.getOnlineServersList()).thenReturn(
211 new ArrayList<ServerName>(onlineServers.keySet()));
212
213 Mockito.when(serverManager.createDestinationServersList()).thenReturn(
214 new ArrayList<ServerName>(onlineServers.keySet()));
215 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(
216 new ArrayList<ServerName>(onlineServers.keySet()));
217
218 for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) {
219 Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true);
220 Mockito.when(serverManager.sendRegionClose(entry.getValue(),
221 entry.getKey(), -1)).thenReturn(true);
222 Mockito.when(serverManager.sendRegionOpen(entry.getValue(),
223 entry.getKey(), -1, null)).thenReturn(RegionOpeningState.OPENED);
224 Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true);
225 }
226
227 Mockito.when(master.getServerManager()).thenReturn(serverManager);
228
229 drainedServers.add(SERVERNAME_A);
230 drainedServers.add(SERVERNAME_B);
231 drainedServers.add(SERVERNAME_C);
232 drainedServers.add(SERVERNAME_D);
233
234 am = new AssignmentManager(server, serverManager, catalogTracker,
235 balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null);
236
237 Mockito.when(master.getAssignmentManager()).thenReturn(am);
238
239 zkWatcher.registerListener(am);
240
241 for (ServerName drained : drainedServers) {
242 addServerToDrainedList(drained, onlineServers, serverManager);
243 }
244
245 am.assign(bulk);
246
247 Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
248 for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) {
249 setRegionOpenedOnZK(zkWatcher, entry.getValue().getServerName(),
250 entry.getValue().getRegion());
251 }
252
253 am.waitForAssignment(REGIONINFO_A);
254 am.waitForAssignment(REGIONINFO_B);
255 am.waitForAssignment(REGIONINFO_C);
256 am.waitForAssignment(REGIONINFO_D);
257 am.waitForAssignment(REGIONINFO_E);
258
259 Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments();
260 for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) {
261 LOG.info("Region Assignment: "
262 + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue());
263 bunchServersAssigned.add(entry.getValue());
264 }
265
266 for (ServerName sn : drainedServers) {
267 assertFalse(bunchServersAssigned.contains(sn));
268 }
269 }
270
271 private void addServerToDrainedList(ServerName serverName,
272 Map<ServerName, ServerLoad> onlineServers, ServerManager serverManager) {
273 onlineServers.remove(serverName);
274 List<ServerName> availableServers = new ArrayList<ServerName>(onlineServers.keySet());
275 Mockito.when(serverManager.createDestinationServersList()).thenReturn(availableServers);
276 Mockito.when(serverManager.createDestinationServersList(null)).thenReturn(availableServers);
277 }
278
279 private void setRegionOpenedOnZK(final ZooKeeperWatcher zkWatcher, final ServerName serverName,
280 HRegionInfo hregionInfo) throws Exception {
281 int version = ZKAssign.getVersion(zkWatcher, hregionInfo);
282 int versionTransition = ZKAssign.transitionNode(zkWatcher,
283 hregionInfo, serverName, EventType.M_ZK_REGION_OFFLINE,
284 EventType.RS_ZK_REGION_OPENING, version);
285 ZKAssign.transitionNodeOpened(zkWatcher, hregionInfo, serverName, versionTransition);
286 }
287
288 private ExecutorService startupMasterExecutor(final String name) {
289 ExecutorService executor = new ExecutorService(name);
290 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
291 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
292 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
293 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
294 return executor;
295 }
296 }