1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23
24 import org.apache.hadoop.hbase.HBaseTestingUtility;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.HRegionInfo;
27 import org.apache.hadoop.hbase.HTableDescriptor;
28 import org.apache.hadoop.hbase.testclassification.MediumTests;
29 import org.apache.hadoop.hbase.NotServingRegionException;
30 import org.apache.hadoop.hbase.ServerName;
31 import org.apache.hadoop.hbase.MetaTableAccessor;
32 import org.apache.hadoop.hbase.TableName;
33 import org.apache.hadoop.hbase.client.HTable;
34 import org.apache.hadoop.hbase.client.Put;
35 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
36 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
37 import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
38 import org.apache.hadoop.hbase.executor.EventType;
39 import org.apache.hadoop.hbase.protobuf.RequestConverter;
40 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
41 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
42 import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
43 import org.apache.hadoop.hbase.util.Threads;
44 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
45 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
46 import org.apache.zookeeper.KeeperException;
47 import org.apache.zookeeper.KeeperException.NodeExistsException;
48 import org.junit.After;
49 import org.junit.AfterClass;
50 import org.junit.Assert;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54 import org.mortbay.log.Log;
55
56 import com.google.protobuf.ServiceException;
57
58
59
60
61
62 @Category(MediumTests.class)
63 public class TestRegionServerNoMaster {
64
65 private static final int NB_SERVERS = 1;
66 private static HTable table;
67 private static final byte[] row = "ee".getBytes();
68
69 private static HRegionInfo hri;
70
71 private static byte[] regionName;
72 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
73
74
75 @BeforeClass
76 public static void before() throws Exception {
77 HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true);
78 HTU.startMiniCluster(NB_SERVERS);
79 final TableName tableName = TableName.valueOf(TestRegionServerNoMaster.class.getSimpleName());
80
81
82 table = HTU.createTable(tableName,HConstants.CATALOG_FAMILY);
83 Put p = new Put(row);
84 p.add(HConstants.CATALOG_FAMILY, row, row);
85 table.put(p);
86
87 hri = table.getRegionLocation(row, false).getRegionInfo();
88 regionName = hri.getRegionName();
89
90 stopMasterAndAssignMeta(HTU);
91 }
92
93 public static void stopMasterAndAssignMeta(HBaseTestingUtility HTU)
94 throws NodeExistsException, KeeperException, IOException, InterruptedException {
95
96 HTU.getHBaseCluster().getMaster().stopMaster();
97
98 Log.info("Waiting until master thread exits");
99 while (HTU.getHBaseCluster().getMasterThread() != null
100 && HTU.getHBaseCluster().getMasterThread().isAlive()) {
101 Threads.sleep(100);
102 }
103 }
104
105
106 public static void flushRegion(HBaseTestingUtility HTU, HRegionInfo regionInfo) throws IOException {
107 for (RegionServerThread rst : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
108 HRegion region = rst.getRegionServer().getRegionByEncodedName(regionInfo.getEncodedName());
109 if (region != null) {
110 region.flushcache();
111 return;
112 }
113 }
114 throw new IOException("Region to flush cannot be found");
115 }
116
117 @AfterClass
118 public static void afterClass() throws Exception {
119 table.close();
120 HTU.shutdownMiniCluster();
121 }
122
123 @After
124 public void after() throws Exception {
125
126
127
128 ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hri);
129 }
130
131
132 private static HRegionServer getRS() {
133 return HTU.getHBaseCluster().getLiveRegionServerThreads().get(0).getRegionServer();
134 }
135
136
137
138
139
140 private void reopenRegion() throws Exception {
141
142 ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
143
144 AdminProtos.OpenRegionRequest orr =
145 RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, 0, null, null);
146 AdminProtos.OpenRegionResponse responseOpen = getRS().rpcServices.openRegion(null, orr);
147 Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
148 Assert.assertTrue(responseOpen.getOpeningState(0).
149 equals(AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED));
150
151
152 checkRegionIsOpened();
153 }
154
155 private void checkRegionIsOpened() throws Exception {
156
157 while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
158 Thread.sleep(1);
159 }
160
161 Assert.assertTrue(getRS().getRegion(regionName).isAvailable());
162
163 Assert.assertTrue(
164 ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(),
165 getRS().getServerName()));
166 }
167
168
169 private void checkRegionIsClosed() throws Exception {
170
171 while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
172 Thread.sleep(1);
173 }
174
175 try {
176 Assert.assertFalse(getRS().getRegion(regionName).isAvailable());
177 } catch (NotServingRegionException expected) {
178
179 }
180
181
182 }
183
184
185
186
187
188 private void closeNoZK() throws Exception {
189
190 AdminProtos.CloseRegionRequest crr =
191 RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false);
192 AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
193 Assert.assertTrue(responseClose.getClosed());
194
195
196 checkRegionIsClosed();
197 }
198
199
200 @Test(timeout = 60000)
201 public void testCloseByRegionServer() throws Exception {
202 closeNoZK();
203 reopenRegion();
204 }
205
206 @Test(timeout = 60000)
207 public void testCloseByMasterWithoutZNode() throws Exception {
208
209
210 AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(
211 getRS().getServerName(), regionName, true);
212 AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
213 Assert.assertTrue(responseClose.getClosed());
214
215
216 while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
217 Thread.sleep(1);
218 }
219
220
221 Assert.assertTrue("The close should have failed", getRS().getRegion(regionName).isAvailable());
222 }
223
224 @Test(timeout = 60000)
225 public void testOpenCloseByMasterWithZNode() throws Exception {
226
227 ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
228
229 AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(
230 getRS().getServerName(), regionName, true);
231 AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
232 Assert.assertTrue(responseClose.getClosed());
233
234 checkRegionIsClosed();
235
236 ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(),
237 getRS().getServerName());
238
239 reopenRegion();
240 }
241
242
243
244
245
246
247
248
249
250
251
252 @Test(timeout = 60000)
253 public void testMultipleOpen() throws Exception {
254
255
256 closeNoZK();
257 checkRegionIsClosed();
258
259
260 ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
261
262
263 for (int i = 0; i < 10; i++) {
264 AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(
265 getRS().getServerName(), hri, 0, null, null);
266 AdminProtos.OpenRegionResponse responseOpen = getRS().rpcServices.openRegion(null, orr);
267 Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
268
269 AdminProtos.OpenRegionResponse.RegionOpeningState ors = responseOpen.getOpeningState(0);
270 Assert.assertTrue("request " + i + " failed",
271 ors.equals(AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED) ||
272 ors.equals(AdminProtos.OpenRegionResponse.RegionOpeningState.ALREADY_OPENED)
273 );
274 }
275
276 checkRegionIsOpened();
277 }
278
279 @Test
280 public void testOpenClosingRegion() throws Exception {
281 Assert.assertTrue(getRS().getRegion(regionName).isAvailable());
282
283 try {
284
285 ServerName sn = getRS().getServerName();
286 MetaTableAccessor.updateRegionLocation(getRS().getConnection(),
287 hri, sn, getRS().getRegion(regionName).getOpenSeqNum());
288
289 getRS().regionsInTransitionInRS.put(hri.getEncodedNameAsBytes(), Boolean.FALSE);
290 AdminProtos.OpenRegionRequest orr =
291 RequestConverter.buildOpenRegionRequest(sn, hri, 0, null, null);
292 getRS().rpcServices.openRegion(null, orr);
293 Assert.fail("The closing region should not be opened");
294 } catch (ServiceException se) {
295 Assert.assertTrue("The region should be already in transition",
296 se.getCause() instanceof RegionAlreadyInTransitionException);
297 } finally {
298 getRS().regionsInTransitionInRS.remove(hri.getEncodedNameAsBytes());
299 }
300 }
301
302 @Test(timeout = 60000)
303 public void testMultipleCloseFromMaster() throws Exception {
304
305
306 ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
307 for (int i = 0; i < 10; i++) {
308 AdminProtos.CloseRegionRequest crr =
309 RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, 0, null, true);
310 try {
311 AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
312 Assert.assertEquals("The first request should succeeds", 0, i);
313 Assert.assertTrue("request " + i + " failed",
314 responseClose.getClosed() || responseClose.hasClosed());
315 } catch (ServiceException se) {
316 Assert.assertTrue("The next queries should throw an exception.", i > 0);
317 }
318 }
319
320 checkRegionIsClosed();
321
322 Assert.assertTrue(
323 ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(),
324 getRS().getServerName())
325 );
326
327 reopenRegion();
328 }
329
330
331
332
333 @Test(timeout = 60000)
334 public void testCancelOpeningWithoutZK() throws Exception {
335
336 closeNoZK();
337 checkRegionIsClosed();
338
339
340 ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
341 getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE);
342
343
344 AdminProtos.CloseRegionRequest crr =
345 RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false);
346 try {
347 getRS().rpcServices.closeRegion(null, crr);
348 Assert.assertTrue(false);
349 } catch (ServiceException expected) {
350 }
351
352
353 Assert.assertEquals(Boolean.FALSE, getRS().getRegionsInTransitionInRS().get(
354 hri.getEncodedNameAsBytes()));
355
356
357 HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable());
358
359 BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager();
360 csm.initialize(getRS());
361 csm.start();
362
363 ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
364 new ZkOpenRegionCoordination.ZkOpenRegionDetails();
365 zkCrd.setServerName(getRS().getServerName());
366 zkCrd.setVersionOfOfflineNode(0);
367
368 getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd,
369 csm.getOpenRegionCoordination(), zkCrd));
370
371
372 checkRegionIsClosed();
373
374
375 Assert.assertTrue(ZKAssign.deleteNode(
376 getRS().getZooKeeper(), hri.getEncodedName(),
377 EventType.RS_ZK_REGION_FAILED_OPEN, 1)
378 );
379
380 reopenRegion();
381 }
382
383
384
385
386
387 @Test(timeout = 60000)
388 public void testCancelOpeningWithZK() throws Exception {
389
390 closeNoZK();
391 checkRegionIsClosed();
392
393
394 getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE);
395
396
397 ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
398 AdminProtos.CloseRegionRequest crr =
399 RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false);
400 try {
401 getRS().rpcServices.closeRegion(null, crr);
402 Assert.assertTrue(false);
403 } catch (ServiceException expected) {
404 Assert.assertTrue(expected.getCause() instanceof RegionAlreadyInTransitionException);
405 }
406
407
408 Assert.assertTrue(ZKAssign.deleteNode(
409 getRS().getZooKeeper(), hri.getEncodedName(),
410 EventType.M_ZK_REGION_CLOSING, 0)
411 );
412
413
414 Assert.assertEquals(Boolean.FALSE, getRS().getRegionsInTransitionInRS().get(
415 hri.getEncodedNameAsBytes()));
416
417
418
419
420
421
422 HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable());
423
424 BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager();
425 csm.initialize(getRS());
426 csm.start();
427
428 ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
429 new ZkOpenRegionCoordination.ZkOpenRegionDetails();
430 zkCrd.setServerName(getRS().getServerName());
431 zkCrd.setVersionOfOfflineNode(0);
432
433 getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd,
434 csm.getOpenRegionCoordination(), zkCrd));
435
436
437 checkRegionIsClosed();
438
439
440 Assert.assertEquals(-1, ZKAssign.getVersion(HTU.getZooKeeperWatcher(), hri));
441
442 reopenRegion();
443 }
444
445
446
447
448
449 @Test
450 public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
451 Assert.assertTrue(getRS().getRegion(regionName).isAvailable());
452
453 ServerName sn = getRS().getServerName();
454 ServerName earlierServerName = ServerName.valueOf(sn.getHostname(), sn.getPort(), 1);
455
456 try {
457 CloseRegionRequest request = RequestConverter.buildCloseRegionRequest(earlierServerName, regionName, true);
458 getRS().getRSRpcServices().closeRegion(null, request);
459 Assert.fail("The closeRegion should have been rejected");
460 } catch (ServiceException se) {
461 Assert.assertTrue(se.getCause() instanceof IOException);
462 Assert.assertTrue(se.getCause().getMessage().contains("This RPC was intended for a different server"));
463 }
464
465
466 closeNoZK();
467 try {
468 AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(
469 earlierServerName, hri, 0, null, null);
470 getRS().getRSRpcServices().openRegion(null, orr);
471 Assert.fail("The openRegion should have been rejected");
472 } catch (ServiceException se) {
473 Assert.assertTrue(se.getCause() instanceof IOException);
474 Assert.assertTrue(se.getCause().getMessage().contains("This RPC was intended for a different server"));
475 } finally {
476 reopenRegion();
477 }
478 }
479 }