1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.catalog;
21  
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.net.ConnectException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.concurrent.CountDownLatch;
29  import java.util.concurrent.atomic.AtomicBoolean;
30  import java.util.concurrent.atomic.AtomicInteger;
31  
32  import junit.framework.Assert;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.hbase.*;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.HConnection;
39  import org.apache.hadoop.hbase.client.HConnectionManager;
40  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
41  import org.apache.hadoop.hbase.client.Result;
42  import org.apache.hadoop.hbase.client.RetriesExhaustedException;
43  import org.apache.hadoop.hbase.client.ServerCallable;
44  import org.apache.hadoop.hbase.ipc.HRegionInterface;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.Threads;
47  import org.apache.hadoop.hbase.util.Writables;
48  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
49  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
50  import org.apache.hadoop.util.Progressable;
51  import org.apache.zookeeper.KeeperException;
52  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
53  import org.junit.After;
54  import org.junit.AfterClass;
55  import org.junit.Before;
56  import org.junit.BeforeClass;
57  import org.junit.Ignore;
58  import org.junit.Test;
59  import org.junit.experimental.categories.Category;
60  import org.mockito.Mockito;
61  
62  /**
63   * Test {@link CatalogTracker}
64   */
65  @Category(MediumTests.class)
66  public class TestCatalogTracker {
67    private static final Log LOG = LogFactory.getLog(TestCatalogTracker.class);
68    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
69    private static final ServerName SN =
70      new ServerName("example.org", 1234, System.currentTimeMillis());
71    private ZooKeeperWatcher watcher;
72    private Abortable abortable;
73  
74    @BeforeClass public static void beforeClass() throws Exception {
75      // Set this down so tests run quicker
76      UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
77      UTIL.startMiniZKCluster();
78    }
79  
80    @AfterClass public static void afterClass() throws IOException {
81      UTIL.getZkCluster().shutdown();
82    }
83  
84    @Before public void before() throws IOException {
85      this.abortable = new Abortable() {
86        @Override
87        public void abort(String why, Throwable e) {
88          LOG.info(why, e);
89        }
90        
91        @Override
92        public boolean isAborted()  {
93          return false;
94        }
95      };
96      this.watcher = new ZooKeeperWatcher(UTIL.getConfiguration(),
97        this.getClass().getSimpleName(), this.abortable, true);
98    }
99  
100   @After public void after() {
101     this.watcher.close();
102   }
103 
104   private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
105   throws IOException, InterruptedException {
106     CatalogTracker ct = new CatalogTracker(this.watcher, UTIL.getConfiguration(),
107       c, this.abortable);
108     ct.start();
109     return ct;
110   }
111 
112   /**
113    * Test that we get notification if .META. moves.
114    * @throws IOException 
115    * @throws InterruptedException 
116    * @throws KeeperException 
117    */
118   @Test public void testThatIfMETAMovesWeAreNotified()
119   throws IOException, InterruptedException, KeeperException {
120     HConnection connection = Mockito.mock(HConnection.class);
121     constructAndStartCatalogTracker(connection);
122     try {
123       RootLocationEditor.setRootLocation(this.watcher,
124         new ServerName("example.com", 1234, System.currentTimeMillis()));
125     } finally {
126       // Clean out root location or later tests will be confused... they presume
127       // start fresh in zk.
128       RootLocationEditor.deleteRootLocation(this.watcher);
129     }
130   }
131 
132   /**
133    * Test interruptable while blocking wait on root and meta.
134    * @throws IOException
135    * @throws InterruptedException
136    */
137   @Test public void testInterruptWaitOnMetaAndRoot()
138   throws IOException, InterruptedException {
139     HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
140     HConnection connection = mockConnection(implementation);
141     try {
142       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
143       ServerName hsa = ct.getRootLocation();
144       Assert.assertNull(hsa);
145       ServerName meta = ct.getMetaLocation();
146       Assert.assertNull(meta);
147       Thread t = new Thread() {
148         @Override
149         public void run() {
150           try {
151             ct.waitForMeta();
152           } catch (InterruptedException e) {
153             throw new RuntimeException("Interrupted", e);
154           }
155         }
156       };
157       t.start();
158       while (!t.isAlive())
159         Threads.sleep(1);
160       Threads.sleep(1);
161       assertTrue(t.isAlive());
162       ct.stop();
163       // Join the thread... should exit shortly.
164       t.join();
165     } finally {
166       HConnectionManager.deleteConnection(UTIL.getConfiguration());
167     }
168   }
169 
170   /**
171    * Test for HBASE-4288.  Throw an IOE when trying to verify meta region and
172    * prove it doesn't cause master shutdown.
173    * @see <a href="https://issues.apache.org/jira/browse/HBASE-4288">HBASE-4288</a>
174    * @throws IOException
175    * @throws InterruptedException
176    * @throws KeeperException
177    */
178   @Test
179   public void testServerNotRunningIOException()
180   throws IOException, InterruptedException, KeeperException {
181     // Mock an HRegionInterface.
182     final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
183     HConnection connection = mockConnection(implementation);
184     try {
185       // If a 'getRegionInfo' is called on mocked HRegionInterface, throw IOE
186       // the first time.  'Succeed' the second time we are called.
187       Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
188         thenThrow(new IOException("Server not running, aborting")).
189         thenReturn(new HRegionInfo());
190 
191       // After we encounter the above 'Server not running', we should catch the
192       // IOE and go into retrying for the meta mode.  We'll do gets on -ROOT- to
193       // get new meta location.  Return something so this 'get' succeeds
194       // (here we mock up getRegionServerWithRetries, the wrapper around
195       // the actual get).
196 
197       // TODO: Refactor.  This method has been moved out of HConnection.
198       // It works for now but has been deprecated.
199       Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
200         thenReturn(getMetaTableRowResult());
201 
202       // Now start up the catalogtracker with our doctored Connection.
203       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
204       try {
205         // Set a location for root and meta.
206         RootLocationEditor.setRootLocation(this.watcher, SN);
207         ct.setMetaLocation(SN);
208         // Call the method that HBASE-4288 calls.  It will try and verify the
209         // meta location and will fail on first attempt then go into a long wait.
210         // So, do this in a thread and then reset meta location to break it out
211         // of its wait after a bit of time.
212         final AtomicBoolean metaSet = new AtomicBoolean(false);
213         final CountDownLatch latch = new CountDownLatch(1);
214         Thread t = new Thread() {
215           @Override
216           public void run() {
217             try {
218               latch.countDown();
219               metaSet.set(ct.waitForMeta(100000) !=  null);
220             } catch (Exception e) {
221               throw new RuntimeException(e);
222             }
223           }
224         };
225         t.start();
226         latch.await();
227         Threads.sleep(1);
228         // Now reset the meta as though it were redeployed.
229         ct.setMetaLocation(SN);
230         t.join();
231         Assert.assertTrue(metaSet.get());
232       } finally {
233         // Clean out root and meta locations or later tests will be confused...
234         // they presume start fresh in zk.
235         ct.resetMetaLocation();
236         RootLocationEditor.deleteRootLocation(this.watcher);
237       }
238     } finally {
239       // Clear out our doctored connection or could mess up subsequent tests.
240       HConnectionManager.deleteConnection(UTIL.getConfiguration());
241     }
242   }
243 
244   private void testVerifyMetaRegionLocationWithException(Exception ex)
245   throws IOException, InterruptedException, KeeperException {
246     // Mock an HRegionInterface.
247     final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
248     HConnection connection = mockConnection(implementation);
249     try {
250       // If a 'get' is called on mocked interface, throw connection refused.
251       Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).
252         thenThrow(ex);
253       // Now start up the catalogtracker with our doctored Connection.
254       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
255       try {
256         RootLocationEditor.setRootLocation(this.watcher, SN);
257         long timeout = UTIL.getConfiguration().
258           getLong("hbase.catalog.verification.timeout", 1000);
259         Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
260       } finally {
261         // Clean out root location or later tests will be confused... they
262         // presume start fresh in zk.
263         RootLocationEditor.deleteRootLocation(this.watcher);
264       }
265     } finally {
266       // Clear out our doctored connection or could mess up subsequent tests.
267       HConnectionManager.deleteConnection(UTIL.getConfiguration());
268     }
269   }
270 
271   /**
272    * Test we survive a connection refused {@link ConnectException}
273    * @throws IOException
274    * @throws InterruptedException
275    * @throws KeeperException
276    */
277   @Test
278   public void testGetMetaServerConnectionFails()
279   throws IOException, InterruptedException, KeeperException {
280     testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
281   }
282 
283   /**
284    * Test that verifyMetaRegionLocation properly handles getting a
285    * ServerNotRunningException. See HBASE-4470.
286    * Note this doesn't check the exact exception thrown in the
287    * HBASE-4470 as there it is thrown from getHConnection() and
288    * here it is thrown from get() -- but those are both called
289    * from the same function anyway, and this way is less invasive than
290    * throwing from getHConnection would be.
291    *
292    * @throws IOException
293    * @throws InterruptedException
294    * @throws KeeperException
295    */
296   @Test
297   public void testVerifyMetaRegionServerNotRunning()
298   throws IOException, InterruptedException, KeeperException {
299     testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
300   }
301 
302   /**
303    * Test get of root region fails properly if nothing to connect to.
304    * @throws IOException
305    * @throws InterruptedException
306    * @throws KeeperException
307    */
308   @Test
309   public void testVerifyRootRegionLocationFails()
310   throws IOException, InterruptedException, KeeperException {
311     HConnection connection = Mockito.mock(HConnection.class);
312     ConnectException connectException =
313       new ConnectException("Connection refused");
314     final HRegionInterface implementation =
315       Mockito.mock(HRegionInterface.class);
316     Mockito.when(implementation.getRegionInfo((byte [])Mockito.any())).
317       thenThrow(connectException);
318     Mockito.when(connection.getHRegionConnection(Mockito.anyString(),
319       Mockito.anyInt(), Mockito.anyBoolean())).
320       thenReturn(implementation);
321     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
322     try {
323       RootLocationEditor.setRootLocation(this.watcher,
324         new ServerName("example.com", 1234, System.currentTimeMillis()));
325       Assert.assertFalse(ct.verifyRootRegionLocation(100));
326     } finally {
327       // Clean out root location or later tests will be confused... they presume
328       // start fresh in zk.
329       RootLocationEditor.deleteRootLocation(this.watcher);
330     }
331   }
332 
333   @Test (expected = NotAllMetaRegionsOnlineException.class)
334   public void testTimeoutWaitForRoot()
335   throws IOException, InterruptedException {
336     HConnection connection = Mockito.mock(HConnection.class);
337     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
338     ct.waitForRoot(100);
339   }
340 
341   @Test (expected = RetriesExhaustedException.class)
342   public void testTimeoutWaitForMeta()
343   throws IOException, InterruptedException {
344     HConnection connection =
345       HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
346     try {
347       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
348       ct.waitForMeta(100);
349     } finally {
350       HConnectionManager.deleteConnection(UTIL.getConfiguration());
351     }
352   }
353 
354   /**
355    * Test waiting on root w/ no timeout specified.
356    * @throws IOException
357    * @throws InterruptedException
358    * @throws KeeperException
359    */
360   @Test public void testNoTimeoutWaitForRoot()
361   throws IOException, InterruptedException, KeeperException {
362     HConnection connection = Mockito.mock(HConnection.class);
363     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
364     ServerName hsa = ct.getRootLocation();
365     Assert.assertNull(hsa);
366 
367     // Now test waiting on root location getting set.
368     Thread t = new WaitOnMetaThread(ct);
369     startWaitAliveThenWaitItLives(t, 1000);
370     // Set a root location.
371     hsa = setRootLocation();
372     // Join the thread... should exit shortly.
373     t.join();
374     // Now root is available.
375     Assert.assertTrue(ct.getRootLocation().equals(hsa));
376   }
377 
378   private ServerName setRootLocation() throws KeeperException {
379     RootLocationEditor.setRootLocation(this.watcher, SN);
380     return SN;
381   }
382 
383   /**
384    * Test waiting on meta w/ no timeout specified.
385    * @throws Exception 
386    */
387   @Ignore // Can't make it work reliably on all platforms; mockito gets confused
388   // Throwing: org.mockito.exceptions.misusing.WrongTypeOfReturnValue:
389   // Result cannot be returned by locateRegion()
390   // If you plug locateRegion, it then throws for incCounter, and if you plug
391   // that ... and so one.
392   @Test public void testNoTimeoutWaitForMeta()
393   throws Exception {
394     // Mock an HConnection and a HRegionInterface implementation.  Have the
395     // HConnection return the HRI.  Have the HRI return a few mocked up responses
396     // to make our test work.
397     // Mock an HRegionInterface.
398     final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
399     HConnection connection = mockConnection(implementation);
400     try {
401       // Now the ct is up... set into the mocks some answers that make it look
402       // like things have been getting assigned. Make it so we'll return a
403       // location (no matter what the Get is). Same for getHRegionInfo -- always
404       // just return the meta region.
405       final Result result = getMetaTableRowResult();
406 
407       // TODO: Refactor.  This method has been moved out of HConnection.
408       // It works for now but has been deprecated.
409       Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
410         thenReturn(result);
411       Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
412         thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
413       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
414       ServerName hsa = ct.getMetaLocation();
415       Assert.assertNull(hsa);
416 
417       // Now test waiting on meta location getting set.
418       Thread t = new WaitOnMetaThread(ct) {
419         @Override
420         void doWaiting() throws InterruptedException {
421           this.ct.waitForMeta();
422         }
423       };
424       startWaitAliveThenWaitItLives(t, 1000);
425 
426       // This should trigger wake up of meta wait (Its the removal of the meta
427       // region unassigned node that triggers catalogtrackers that a meta has
428       // been assigned).
429       String node = ct.getMetaNodeTracker().getNode();
430       ZKUtil.createAndFailSilent(this.watcher, node);
431       MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
432       ZKUtil.deleteNode(this.watcher, node);
433       // Go get the new meta location. waitForMeta gets and verifies meta.
434       Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
435       // Join the thread... should exit shortly.
436       t.join();
437       // Now meta is available.
438       Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
439     } finally {
440       HConnectionManager.deleteConnection(UTIL.getConfiguration());
441     }
442   }
443 
444   /**
445    * @param implementation An {@link HRegionInterface} instance; you'll likely
446    * want to pass a mocked HRS; can be null.
447    * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
448    * {@link HConnection#getConfiguration()} is called, a 'location' when
449    * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
450    * and that returns the passed {@link HRegionInterface} instance when
451    * {@link HConnection#getHRegionConnection(String, int)}
452    * is called (Be sure call
453    * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
454    * when done with this mocked Connection.
455    * @throws IOException
456    */
457   private HConnection mockConnection(final HRegionInterface implementation)
458   throws IOException {
459     HConnection connection =
460       HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
461     Mockito.doNothing().when(connection).close();
462     // Make it so we return any old location when asked.
463     final HRegionLocation anyLocation =
464       new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
465         SN.getPort());
466     Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
467         (byte[]) Mockito.any(), Mockito.anyBoolean())).
468       thenReturn(anyLocation);
469     Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
470         (byte[]) Mockito.any())).
471       thenReturn(anyLocation);
472     if (implementation != null) {
473       // If a call to getHRegionConnection, return this implementation.
474       Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
475         thenReturn(implementation);
476     }
477     return connection;
478   }
479 
480   /**
481    * @return A mocked up Result that fakes a Get on a row in the
482    * <code>.META.</code> table.
483    * @throws IOException 
484    */
485   private Result getMetaTableRowResult() throws IOException {
486     List<KeyValue> kvs = new ArrayList<KeyValue>();
487     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
488       HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
489       Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
490     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
491       HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
492       Bytes.toBytes(SN.getHostAndPort())));
493     kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
494       HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
495       Bytes.toBytes(SN.getStartcode())));
496     return new Result(kvs);
497   }
498 
499   private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
500     t.start();
501     while(!t.isAlive()) {
502       // Wait
503     }
504     // Wait one second.
505     Threads.sleep(ms);
506     Assert.assertTrue("Assert " + t.getName() + " still waiting", t.isAlive());
507   }
508 
509   class CountingProgressable implements Progressable {
510     final AtomicInteger counter = new AtomicInteger(0);
511     @Override
512     public void progress() {
513       this.counter.incrementAndGet();
514     }
515   }
516 
517   /**
518    * Wait on META.
519    * Default is wait on -ROOT-.
520    */
521   class WaitOnMetaThread extends Thread {
522     final CatalogTracker ct;
523 
524     WaitOnMetaThread(final CatalogTracker ct) {
525       super("WaitOnMeta");
526       this.ct = ct;
527     }
528 
529     @Override
530     public void run() {
531       try {
532         doWaiting();
533       } catch (InterruptedException e) {
534         throw new RuntimeException("Failed wait", e);
535       }
536       LOG.info("Exiting " + getName());
537     }
538 
539     void doWaiting() throws InterruptedException {
540       this.ct.waitForRoot();
541     }
542   }
543 
544   @org.junit.Rule
545   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
546     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
547 }
548