1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.lang.reflect.InvocationTargetException;
27 import java.net.BindException;
28 import java.util.Collection;
29 import java.util.Set;
30 import java.util.concurrent.CopyOnWriteArraySet;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.HMsg;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HServerAddress;
40 import org.apache.hadoop.hbase.HServerInfo;
41 import org.apache.hadoop.hbase.MiniHBaseCluster;
42 import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
43 import org.apache.hadoop.hbase.client.Get;
44 import org.apache.hadoop.hbase.client.HTable;
45 import org.apache.hadoop.hbase.client.Put;
46 import org.apache.hadoop.hbase.client.Result;
47 import org.apache.hadoop.hbase.client.ResultScanner;
48 import org.apache.hadoop.hbase.client.Scan;
49 import org.apache.hadoop.hbase.regionserver.HRegion;
50 import org.apache.hadoop.hbase.regionserver.HRegionServer;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.hbase.util.Threads;
53 import org.apache.hadoop.hbase.util.Writables;
54 import org.junit.AfterClass;
55 import org.junit.Assert;
56 import org.junit.Before;
57 import org.junit.BeforeClass;
58 import org.junit.Test;
59
60
61
62
63
64 public class TestMasterTransitions {
65 private static final Log LOG = LogFactory.getLog(TestMasterTransitions.class);
66 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67 private static final String TABLENAME = "master_transitions";
68 private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
69 Bytes.toBytes("b"), Bytes.toBytes("c")};
70
71
72
73
74
75 @BeforeClass public static void beforeAllTests() throws Exception {
76 TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
77
78
79
80 TEST_UTIL.getConfiguration().setInt("hbase.regions.percheckin", 2);
81
82 TEST_UTIL.startMiniCluster(2);
83
84 TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
85 HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
86 int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
87 TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
88 addToEachStartKey(countOfRegions);
89 }
90
91 @AfterClass public static void afterAllTests() throws IOException {
92 TEST_UTIL.shutdownMiniCluster();
93 }
94
95 @Before public void setup() throws IOException {
96 TEST_UTIL.ensureSomeRegionServersAvailable(2);
97 }
98
99
100
101
102
103
104
105
106
107 static class HBase2428Listener implements RegionServerOperationListener {
108
109 private final Set<RegionServerOperation> postponed =
110 new CopyOnWriteArraySet<RegionServerOperation>();
111 private boolean done = false;;
112 private boolean metaShutdownReceived = false;
113 private final HServerAddress metaAddress;
114 private final MiniHBaseCluster cluster;
115 private final int otherServerIndex;
116 private final HRegionInfo hri;
117 private int closeCount = 0;
118 static final int SERVER_DURATION = 3 * 1000;
119 static final int CLOSE_DURATION = 1 * 1000;
120
121 HBase2428Listener(final MiniHBaseCluster c, final HServerAddress metaAddress,
122 final HRegionInfo closingHRI, final int otherServerIndex) {
123 this.cluster = c;
124 this.metaAddress = metaAddress;
125 this.hri = closingHRI;
126 this.otherServerIndex = otherServerIndex;
127 }
128
129 @Override
130 public boolean process(final RegionServerOperation op) throws IOException {
131
132
133
134 boolean result = true;
135 if (op instanceof ProcessServerShutdown) {
136 ProcessServerShutdown pss = (ProcessServerShutdown)op;
137 if (pss.getDeadServerAddress().equals(this.metaAddress)) {
138
139 if (!this.postponed.contains(pss)) {
140
141 this.cluster.addMessageToSendRegionServer(this.otherServerIndex,
142 new HMsg(HMsg.Type.MSG_REGION_CLOSE, hri,
143 Bytes.toBytes("Forcing close in test")));
144 this.postponed.add(pss);
145
146 pss.setDelay(SERVER_DURATION);
147 this.metaShutdownReceived = true;
148
149 result = false;
150 }
151 }
152 } else {
153
154 if (isWantedCloseOperation(op) != null) {
155 op.setDelay(CLOSE_DURATION);
156
157 this.closeCount++;
158 }
159 }
160 return result;
161 }
162
163 public void processed(final RegionServerOperation op) {
164 if (isWantedCloseOperation(op) != null) return;
165 this.done = true;
166 }
167
168
169
170
171
172
173 private ProcessRegionClose isWantedCloseOperation(final RegionServerOperation op) {
174
175 if (op instanceof ProcessRegionClose) {
176 ProcessRegionClose c = (ProcessRegionClose)op;
177 if (c.regionInfo.equals(hri)) {
178 return c;
179 }
180 }
181 return null;
182 }
183
184 boolean isDone() {
185 return this.done;
186 }
187
188 boolean isMetaShutdownReceived() {
189 return metaShutdownReceived;
190 }
191
192 int getCloseCount() {
193 return this.closeCount;
194 }
195
196 @Override
197 public boolean process(HServerInfo serverInfo, HMsg incomingMsg) {
198 return true;
199 }
200 }
201
202
203
204
205
206
207 @Test (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428()
208 throws Exception {
209 LOG.info("Running testRegionCloseWhenNoMetaHBase2428");
210 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
211 final HMaster master = cluster.getMaster();
212 int metaIndex = cluster.getServerWithMeta();
213
214 int otherServerIndex = -1;
215 for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
216 if (i == metaIndex) continue;
217 otherServerIndex = i;
218 break;
219 }
220 final HRegionServer otherServer = cluster.getRegionServer(otherServerIndex);
221 final HRegionServer metaHRS = cluster.getRegionServer(metaIndex);
222
223
224 final HRegionInfo hri =
225 otherServer.getOnlineRegions().iterator().next().getRegionInfo();
226
227
228 HBase2428Listener listener = new HBase2428Listener(cluster,
229 metaHRS.getHServerInfo().getServerAddress(), hri, otherServerIndex);
230 master.getRegionServerOperationQueue().
231 registerRegionServerOperationListener(listener);
232 try {
233
234 cluster.abortRegionServer(metaIndex);
235
236
237 while(!listener.metaShutdownReceived) Threads.sleep(100);
238 while(!listener.isDone()) Threads.sleep(10);
239
240
241
242 assertTrue(listener.getCloseCount() > 1);
243 assertTrue(listener.getCloseCount() <
244 ((HBase2428Listener.SERVER_DURATION/HBase2428Listener.CLOSE_DURATION) * 2));
245
246
247 assertRegionIsBackOnline(hri);
248 } finally {
249 master.getRegionServerOperationQueue().
250 unregisterRegionServerOperationListener(listener);
251 }
252 }
253
254
255
256
257
258
259
260 @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413()
261 throws IOException {
262 LOG.info("Running testAddingServerBeforeOldIsDead2413");
263 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
264 int count = count();
265 int metaIndex = cluster.getServerWithMeta();
266 MiniHBaseClusterRegionServer metaHRS =
267 (MiniHBaseClusterRegionServer)cluster.getRegionServer(metaIndex);
268 int port = metaHRS.getServerInfo().getServerAddress().getPort();
269 Configuration c = TEST_UTIL.getConfiguration();
270 String oldPort = c.get(HConstants.REGIONSERVER_PORT, "0");
271 try {
272 LOG.info("KILLED=" + metaHRS);
273 metaHRS.kill();
274 c.set(HConstants.REGIONSERVER_PORT, Integer.toString(port));
275
276
277 HRegionServer hrs = null;
278 while (true) {
279 try {
280 hrs = cluster.startRegionServer().getRegionServer();
281 break;
282 } catch (IOException e) {
283 if (e.getCause() != null && e.getCause() instanceof InvocationTargetException) {
284 InvocationTargetException ee = (InvocationTargetException)e.getCause();
285 if (ee.getCause() != null && ee.getCause() instanceof BindException) {
286 LOG.info("BindException; retrying: " + e.toString());
287 }
288 }
289 }
290 }
291 LOG.info("STARTED=" + hrs);
292
293
294 while (hrs.getOnlineRegions().size() < 3) Threads.sleep(100);
295 LOG.info(hrs.toString() + " has " + hrs.getOnlineRegions().size() +
296 " regions");
297 assertEquals(count, count());
298 } finally {
299 c.set(HConstants.REGIONSERVER_PORT, oldPort);
300 }
301 }
302
303
304
305
306
307
308
309
310
311
312 static class HBase2482Listener implements RegionServerOperationListener {
313 private final HRegionServer victim;
314 private boolean abortSent = false;
315
316 private volatile boolean closed = false;
317
318 private final Collection<HRegion> copyOfOnlineRegions;
319
320
321 private HRegionInfo regionToFind;
322
323 HBase2482Listener(final HRegionServer victim) {
324 this.victim = victim;
325
326
327 this.copyOfOnlineRegions =
328 this.victim.getCopyOfOnlineRegionsSortedBySize().values();
329 }
330
331 @Override
332 public boolean process(HServerInfo serverInfo, HMsg incomingMsg) {
333 if (!victim.getServerInfo().equals(serverInfo) ||
334 this.abortSent || !this.closed) {
335 return true;
336 }
337 if (!incomingMsg.isType(HMsg.Type.MSG_REPORT_PROCESS_OPEN)) return true;
338
339 this.regionToFind = incomingMsg.getRegionInfo();
340 String msg = "ABORTING " + this.victim + " because got a " +
341 HMsg.Type.MSG_REPORT_PROCESS_OPEN + " on this server for " +
342 incomingMsg.getRegionInfo().getRegionNameAsString();
343 this.victim.abort(msg);
344 this.abortSent = true;
345 return true;
346 }
347
348 @Override
349 public boolean process(RegionServerOperation op) throws IOException {
350 return true;
351 }
352
353 @Override
354 public void processed(RegionServerOperation op) {
355 if (this.closed || !(op instanceof ProcessRegionClose)) return;
356 ProcessRegionClose close = (ProcessRegionClose)op;
357 for (HRegion r: this.copyOfOnlineRegions) {
358 if (r.getRegionInfo().equals(close.regionInfo)) {
359
360
361 LOG.info("Found close of " +
362 r.getRegionInfo().getRegionNameAsString() +
363 "; setting close happened flag");
364 this.closed = true;
365 break;
366 }
367 }
368 }
369 }
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385 @Test (timeout=300000) public void testKillRSWithOpeningRegion2482()
386 throws Exception {
387 LOG.info("Running testKillRSWithOpeningRegion2482");
388 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
389 if (cluster.getLiveRegionServerThreads().size() < 2) {
390
391 cluster.startRegionServer();
392 }
393
394
395 int countOfMetaRegions = countOfMetaRegions();
396
397 HMaster m = cluster.getMaster();
398
399 MiniHBaseClusterRegionServer hrs =
400 (MiniHBaseClusterRegionServer)cluster.startRegionServer().getRegionServer();
401 LOG.info("Started new regionserver: " + hrs.toString());
402
403 int minimumRegions =
404 countOfMetaRegions/(cluster.getRegionServerThreads().size() * 2);
405 while (hrs.getOnlineRegions().size() < minimumRegions) Threads.sleep(100);
406
407 HBase2482Listener listener = new HBase2482Listener(hrs);
408 m.getRegionServerOperationQueue().
409 registerRegionServerOperationListener(listener);
410 try {
411
412 closeAllNonCatalogRegions(cluster, hrs);
413
414
415 cluster.addMessageToSendRegionServer(hrs,
416 new HMsg(HMsg.Type.TESTING_MSG_BLOCK_RS));
417
418
419 while (!listener.closed) Threads.sleep(100);
420 LOG.info("Past close");
421
422 while(!listener.abortSent) Threads.sleep(100);
423 LOG.info("Past abort send; waiting on all regions to redeploy");
424
425 assertRegionIsBackOnline(listener.regionToFind);
426 } finally {
427 m.getRegionServerOperationQueue().
428 unregisterRegionServerOperationListener(listener);
429 }
430 }
431
432
433
434
435 private int closeAllNonCatalogRegions(final MiniHBaseCluster cluster,
436 final MiniHBaseCluster.MiniHBaseClusterRegionServer hrs)
437 throws IOException {
438 int countOfRegions = 0;
439 for (HRegion r: hrs.getOnlineRegions()) {
440 if (r.getRegionInfo().isMetaRegion()) continue;
441 cluster.addMessageToSendRegionServer(hrs,
442 new HMsg(HMsg.Type.MSG_REGION_CLOSE, r.getRegionInfo()));
443 LOG.info("Sent close of " + r.getRegionInfo().getRegionNameAsString() +
444 " on " + hrs.toString());
445 countOfRegions++;
446 }
447 return countOfRegions;
448 }
449
450 private void assertRegionIsBackOnline(final HRegionInfo hri)
451 throws IOException {
452
453 byte [] row = getStartKey(hri);
454 HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
455 Get g = new Get(row);
456 assertTrue((t.get(g)).size() > 0);
457 }
458
459
460
461
462
463 private static int countOfMetaRegions()
464 throws IOException {
465 HTable meta = new HTable(TEST_UTIL.getConfiguration(),
466 HConstants.META_TABLE_NAME);
467 int rows = 0;
468 Scan scan = new Scan();
469 scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
470 ResultScanner s = meta.getScanner(scan);
471 for (Result r = null; (r = s.next()) != null;) {
472 byte [] b =
473 r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
474 if (b == null || b.length <= 0) break;
475 rows++;
476 }
477 s.close();
478 return rows;
479 }
480
481
482
483
484
485
486
487
488 private static int addToEachStartKey(final int expected) throws IOException {
489 HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
490 HTable meta = new HTable(TEST_UTIL.getConfiguration(),
491 HConstants.META_TABLE_NAME);
492 int rows = 0;
493 Scan scan = new Scan();
494 scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
495 ResultScanner s = meta.getScanner(scan);
496 for (Result r = null; (r = s.next()) != null;) {
497 byte [] b =
498 r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
499 if (b == null || b.length <= 0) break;
500 HRegionInfo hri = Writables.getHRegionInfo(b);
501
502 byte [] row = getStartKey(hri);
503 Put p = new Put(row);
504 p.add(getTestFamily(), getTestQualifier(), row);
505 t.put(p);
506 rows++;
507 }
508 s.close();
509 Assert.assertEquals(expected, rows);
510 return rows;
511 }
512
513
514
515
516
517 private static int count() throws IOException {
518 HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
519 int rows = 0;
520 Scan scan = new Scan();
521 ResultScanner s = t.getScanner(scan);
522 for (Result r = null; (r = s.next()) != null;) {
523 rows++;
524 }
525 s.close();
526 LOG.info("Counted=" + rows);
527 return rows;
528 }
529
530
531
532
533
534 private static byte [] getStartKey(final HRegionInfo hri) {
535 return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
536 Bytes.toBytes("aaa"): hri.getStartKey();
537 }
538
539 private static byte [] getTestFamily() {
540 return FAMILIES[0];
541 }
542
543 private static byte [] getTestQualifier() {
544 return getTestFamily();
545 }
546 }