1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.catalog;
21
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.net.ConnectException;
26 import java.util.ArrayList;
27 import java.util.List;
28 import java.util.concurrent.CountDownLatch;
29 import java.util.concurrent.atomic.AtomicBoolean;
30 import java.util.concurrent.atomic.AtomicInteger;
31
32 import junit.framework.Assert;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.hbase.*;
37 import org.apache.hadoop.hbase.client.Get;
38 import org.apache.hadoop.hbase.client.HConnection;
39 import org.apache.hadoop.hbase.client.HConnectionManager;
40 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
41 import org.apache.hadoop.hbase.client.Result;
42 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
43 import org.apache.hadoop.hbase.client.ServerCallable;
44 import org.apache.hadoop.hbase.ipc.HRegionInterface;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.Threads;
47 import org.apache.hadoop.hbase.util.Writables;
48 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
49 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
50 import org.apache.hadoop.util.Progressable;
51 import org.apache.zookeeper.KeeperException;
52 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
53 import org.junit.After;
54 import org.junit.AfterClass;
55 import org.junit.Before;
56 import org.junit.BeforeClass;
57 import org.junit.Ignore;
58 import org.junit.Test;
59 import org.junit.experimental.categories.Category;
60 import org.mockito.Mockito;
61
62
63
64
65 @Category(MediumTests.class)
66 public class TestCatalogTracker {
67 private static final Log LOG = LogFactory.getLog(TestCatalogTracker.class);
68 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
69 private static final ServerName SN =
70 new ServerName("example.org", 1234, System.currentTimeMillis());
71 private ZooKeeperWatcher watcher;
72 private Abortable abortable;
73
74 @BeforeClass public static void beforeClass() throws Exception {
75
76 UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
77 UTIL.startMiniZKCluster();
78 }
79
80 @AfterClass public static void afterClass() throws IOException {
81 UTIL.getZkCluster().shutdown();
82 }
83
84 @Before public void before() throws IOException {
85 this.abortable = new Abortable() {
86 @Override
87 public void abort(String why, Throwable e) {
88 LOG.info(why, e);
89 }
90
91 @Override
92 public boolean isAborted() {
93 return false;
94 }
95 };
96 this.watcher = new ZooKeeperWatcher(UTIL.getConfiguration(),
97 this.getClass().getSimpleName(), this.abortable, true);
98 }
99
100 @After public void after() {
101 this.watcher.close();
102 }
103
104 private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
105 throws IOException, InterruptedException {
106 CatalogTracker ct = new CatalogTracker(this.watcher, UTIL.getConfiguration(),
107 c, this.abortable);
108 ct.start();
109 return ct;
110 }
111
112
113
114
115
116
117
118 @Test public void testThatIfMETAMovesWeAreNotified()
119 throws IOException, InterruptedException, KeeperException {
120 HConnection connection = Mockito.mock(HConnection.class);
121 constructAndStartCatalogTracker(connection);
122 try {
123 RootLocationEditor.setRootLocation(this.watcher,
124 new ServerName("example.com", 1234, System.currentTimeMillis()));
125 } finally {
126
127
128 RootLocationEditor.deleteRootLocation(this.watcher);
129 }
130 }
131
132
133
134
135
136
137 @Test public void testInterruptWaitOnMetaAndRoot()
138 throws IOException, InterruptedException {
139 HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
140 HConnection connection = mockConnection(implementation);
141 try {
142 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
143 ServerName hsa = ct.getRootLocation();
144 Assert.assertNull(hsa);
145 ServerName meta = ct.getMetaLocation();
146 Assert.assertNull(meta);
147 Thread t = new Thread() {
148 @Override
149 public void run() {
150 try {
151 ct.waitForMeta();
152 } catch (InterruptedException e) {
153 throw new RuntimeException("Interrupted", e);
154 }
155 }
156 };
157 t.start();
158 while (!t.isAlive())
159 Threads.sleep(1);
160 Threads.sleep(1);
161 assertTrue(t.isAlive());
162 ct.stop();
163
164 t.join();
165 } finally {
166 HConnectionManager.deleteConnection(UTIL.getConfiguration());
167 }
168 }
169
170
171
172
173
174
175
176
177
178 @Test
179 public void testServerNotRunningIOException()
180 throws IOException, InterruptedException, KeeperException {
181
182 final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
183 HConnection connection = mockConnection(implementation);
184 try {
185
186
187 Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
188 thenThrow(new IOException("Server not running, aborting")).
189 thenReturn(new HRegionInfo());
190
191
192
193
194
195
196
197
198
199 Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
200 thenReturn(getMetaTableRowResult());
201
202
203 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
204 try {
205
206 RootLocationEditor.setRootLocation(this.watcher, SN);
207 ct.setMetaLocation(SN);
208
209
210
211
212 final AtomicBoolean metaSet = new AtomicBoolean(false);
213 final CountDownLatch latch = new CountDownLatch(1);
214 Thread t = new Thread() {
215 @Override
216 public void run() {
217 try {
218 latch.countDown();
219 metaSet.set(ct.waitForMeta(100000) != null);
220 } catch (Exception e) {
221 throw new RuntimeException(e);
222 }
223 }
224 };
225 t.start();
226 latch.await();
227 Threads.sleep(1);
228
229 ct.setMetaLocation(SN);
230 t.join();
231 Assert.assertTrue(metaSet.get());
232 } finally {
233
234
235 ct.resetMetaLocation();
236 RootLocationEditor.deleteRootLocation(this.watcher);
237 }
238 } finally {
239
240 HConnectionManager.deleteConnection(UTIL.getConfiguration());
241 }
242 }
243
244 private void testVerifyMetaRegionLocationWithException(Exception ex)
245 throws IOException, InterruptedException, KeeperException {
246
247 final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
248 HConnection connection = mockConnection(implementation);
249 try {
250
251 Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).
252 thenThrow(ex);
253
254 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
255 try {
256 RootLocationEditor.setRootLocation(this.watcher, SN);
257 long timeout = UTIL.getConfiguration().
258 getLong("hbase.catalog.verification.timeout", 1000);
259 Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
260 } finally {
261
262
263 RootLocationEditor.deleteRootLocation(this.watcher);
264 }
265 } finally {
266
267 HConnectionManager.deleteConnection(UTIL.getConfiguration());
268 }
269 }
270
271
272
273
274
275
276
277 @Test
278 public void testGetMetaServerConnectionFails()
279 throws IOException, InterruptedException, KeeperException {
280 testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
281 }
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296 @Test
297 public void testVerifyMetaRegionServerNotRunning()
298 throws IOException, InterruptedException, KeeperException {
299 testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
300 }
301
302
303
304
305
306
307
308 @Test
309 public void testVerifyRootRegionLocationFails()
310 throws IOException, InterruptedException, KeeperException {
311 HConnection connection = Mockito.mock(HConnection.class);
312 ConnectException connectException =
313 new ConnectException("Connection refused");
314 final HRegionInterface implementation =
315 Mockito.mock(HRegionInterface.class);
316 Mockito.when(implementation.getRegionInfo((byte [])Mockito.any())).
317 thenThrow(connectException);
318 Mockito.when(connection.getHRegionConnection(Mockito.anyString(),
319 Mockito.anyInt(), Mockito.anyBoolean())).
320 thenReturn(implementation);
321 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
322 try {
323 RootLocationEditor.setRootLocation(this.watcher,
324 new ServerName("example.com", 1234, System.currentTimeMillis()));
325 Assert.assertFalse(ct.verifyRootRegionLocation(100));
326 } finally {
327
328
329 RootLocationEditor.deleteRootLocation(this.watcher);
330 }
331 }
332
333 @Test (expected = NotAllMetaRegionsOnlineException.class)
334 public void testTimeoutWaitForRoot()
335 throws IOException, InterruptedException {
336 HConnection connection = Mockito.mock(HConnection.class);
337 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
338 ct.waitForRoot(100);
339 }
340
341 @Test (expected = RetriesExhaustedException.class)
342 public void testTimeoutWaitForMeta()
343 throws IOException, InterruptedException {
344 HConnection connection =
345 HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
346 try {
347 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
348 ct.waitForMeta(100);
349 } finally {
350 HConnectionManager.deleteConnection(UTIL.getConfiguration());
351 }
352 }
353
354
355
356
357
358
359
360 @Test public void testNoTimeoutWaitForRoot()
361 throws IOException, InterruptedException, KeeperException {
362 HConnection connection = Mockito.mock(HConnection.class);
363 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
364 ServerName hsa = ct.getRootLocation();
365 Assert.assertNull(hsa);
366
367
368 Thread t = new WaitOnMetaThread(ct);
369 startWaitAliveThenWaitItLives(t, 1000);
370
371 hsa = setRootLocation();
372
373 t.join();
374
375 Assert.assertTrue(ct.getRootLocation().equals(hsa));
376 }
377
378 private ServerName setRootLocation() throws KeeperException {
379 RootLocationEditor.setRootLocation(this.watcher, SN);
380 return SN;
381 }
382
383
384
385
386
387 @Ignore
388
389
390
391
392 @Test public void testNoTimeoutWaitForMeta()
393 throws Exception {
394
395
396
397
398 final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
399 HConnection connection = mockConnection(implementation);
400 try {
401
402
403
404
405 final Result result = getMetaTableRowResult();
406
407
408
409 Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
410 thenReturn(result);
411 Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
412 thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
413 final CatalogTracker ct = constructAndStartCatalogTracker(connection);
414 ServerName hsa = ct.getMetaLocation();
415 Assert.assertNull(hsa);
416
417
418 Thread t = new WaitOnMetaThread(ct) {
419 @Override
420 void doWaiting() throws InterruptedException {
421 this.ct.waitForMeta();
422 }
423 };
424 startWaitAliveThenWaitItLives(t, 1000);
425
426
427
428
429 String node = ct.getMetaNodeTracker().getNode();
430 ZKUtil.createAndFailSilent(this.watcher, node);
431 MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
432 ZKUtil.deleteNode(this.watcher, node);
433
434 Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
435
436 t.join();
437
438 Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
439 } finally {
440 HConnectionManager.deleteConnection(UTIL.getConfiguration());
441 }
442 }
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457 private HConnection mockConnection(final HRegionInterface implementation)
458 throws IOException {
459 HConnection connection =
460 HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
461 Mockito.doNothing().when(connection).close();
462
463 final HRegionLocation anyLocation =
464 new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
465 SN.getPort());
466 Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
467 (byte[]) Mockito.any(), Mockito.anyBoolean())).
468 thenReturn(anyLocation);
469 Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
470 (byte[]) Mockito.any())).
471 thenReturn(anyLocation);
472 if (implementation != null) {
473
474 Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
475 thenReturn(implementation);
476 }
477 return connection;
478 }
479
480
481
482
483
484
485 private Result getMetaTableRowResult() throws IOException {
486 List<KeyValue> kvs = new ArrayList<KeyValue>();
487 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
488 HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
489 Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
490 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
491 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
492 Bytes.toBytes(SN.getHostAndPort())));
493 kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
494 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
495 Bytes.toBytes(SN.getStartcode())));
496 return new Result(kvs);
497 }
498
499 private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
500 t.start();
501 while(!t.isAlive()) {
502
503 }
504
505 Threads.sleep(ms);
506 Assert.assertTrue("Assert " + t.getName() + " still waiting", t.isAlive());
507 }
508
509 class CountingProgressable implements Progressable {
510 final AtomicInteger counter = new AtomicInteger(0);
511 @Override
512 public void progress() {
513 this.counter.incrementAndGet();
514 }
515 }
516
517
518
519
520
521 class WaitOnMetaThread extends Thread {
522 final CatalogTracker ct;
523
524 WaitOnMetaThread(final CatalogTracker ct) {
525 super("WaitOnMeta");
526 this.ct = ct;
527 }
528
529 @Override
530 public void run() {
531 try {
532 doWaiting();
533 } catch (InterruptedException e) {
534 throw new RuntimeException("Failed wait", e);
535 }
536 LOG.info("Exiting " + getName());
537 }
538
539 void doWaiting() throws InterruptedException {
540 this.ct.waitForRoot();
541 }
542 }
543
544 @org.junit.Rule
545 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
546 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
547 }
548