View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.catalog;
20  
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.net.ConnectException;
25  import java.util.concurrent.atomic.AtomicInteger;
26  
27  import junit.framework.Assert;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.Abortable;
33  import org.apache.hadoop.hbase.TableName;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HRegionLocation;
38  import org.apache.hadoop.hbase.MediumTests;
39  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
40  import org.apache.hadoop.hbase.ServerName;
41  import org.apache.hadoop.hbase.client.HConnection;
42  import org.apache.hadoop.hbase.client.HConnectionManager;
43  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
44  import org.apache.hadoop.hbase.client.Result;
45  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
46  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
47  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
48  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
49  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
50  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
51  import org.apache.hadoop.hbase.util.Threads;
52  import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
53  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
54  import org.apache.hadoop.util.Progressable;
55  import org.apache.zookeeper.KeeperException;
56  import org.junit.After;
57  import org.junit.AfterClass;
58  import org.junit.Before;
59  import org.junit.BeforeClass;
60  import org.junit.Test;
61  import org.junit.experimental.categories.Category;
62  import org.mockito.Mockito;
63  
64  import com.google.protobuf.RpcController;
65  import com.google.protobuf.ServiceException;
66  
67  /**
68   * Test {@link CatalogTracker}
69   */
70  @Category(MediumTests.class)
71  public class TestCatalogTracker {
72    private static final Log LOG = LogFactory.getLog(TestCatalogTracker.class);
73    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
74    private static final ServerName SN =
75      new ServerName("example.org", 1234, System.currentTimeMillis());
76    private ZooKeeperWatcher watcher;
77    private Abortable abortable;
78  
79    @BeforeClass public static void beforeClass() throws Exception {
80      // Set this down so tests run quicker
81      UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
82      UTIL.startMiniZKCluster();
83    }
84  
85    @AfterClass public static void afterClass() throws IOException {
86      UTIL.getZkCluster().shutdown();
87    }
88  
89    @Before public void before() throws IOException {
90      this.abortable = new Abortable() {
91        @Override
92        public void abort(String why, Throwable e) {
93          LOG.info(why, e);
94        }
95  
96        @Override
97        public boolean isAborted()  {
98          return false;
99        }
100     };
101     this.watcher = new ZooKeeperWatcher(UTIL.getConfiguration(),
102       this.getClass().getSimpleName(), this.abortable, true);
103   }
104 
105   @After public void after() {
106     this.watcher.close();
107   }
108 
109   private CatalogTracker constructAndStartCatalogTracker(final HConnection c)
110   throws IOException, InterruptedException {
111     CatalogTracker ct = new CatalogTracker(this.watcher, UTIL.getConfiguration(),
112       c, this.abortable);
113     ct.start();
114     return ct;
115   }
116 
117   /**
118    * Test that we get notification if .META. moves.
119    * @throws IOException
120    * @throws InterruptedException
121    * @throws KeeperException
122    */
123   @Test public void testThatIfMETAMovesWeAreNotified()
124   throws IOException, InterruptedException, KeeperException {
125     HConnection connection = Mockito.mock(HConnection.class);
126     constructAndStartCatalogTracker(connection);
127     try {
128       MetaRegionTracker.setMetaLocation(this.watcher,
129           new ServerName("example.com", 1234, System.currentTimeMillis()));
130     } finally {
131       // Clean out meta location or later tests will be confused... they presume
132       // start fresh in zk.
133       MetaRegionTracker.deleteMetaLocation(this.watcher);
134     }
135   }
136 
137   /**
138    * Test interruptable while blocking wait on meta.
139    * @throws IOException
140    * @throws ServiceException
141    * @throws InterruptedException
142    */
143   @Test public void testInterruptWaitOnMeta()
144   throws IOException, InterruptedException, ServiceException {
145     final ClientProtos.ClientService.BlockingInterface client =
146       Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
147     HConnection connection = mockConnection(null, client);
148     try {
149       Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
150       thenReturn(GetResponse.newBuilder().build());
151       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
152       ServerName meta = ct.getMetaLocation();
153       Assert.assertNull(meta);
154       Thread t = new Thread() {
155         @Override
156         public void run() {
157           try {
158             ct.waitForMeta();
159           } catch (InterruptedException e) {
160             throw new RuntimeException("Interrupted", e);
161           }
162         }
163       };
164       t.start();
165       while (!t.isAlive())
166         Threads.sleep(1);
167       Threads.sleep(1);
168       assertTrue(t.isAlive());
169       ct.stop();
170       // Join the thread... should exit shortly.
171       t.join();
172     } finally {
173       HConnectionManager.deleteConnection(UTIL.getConfiguration());
174     }
175   }
176 
177   private void testVerifyMetaRegionLocationWithException(Exception ex)
178   throws IOException, InterruptedException, KeeperException, ServiceException {
179     // Mock an ClientProtocol.
180     final ClientProtos.ClientService.BlockingInterface implementation =
181       Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
182     HConnection connection = mockConnection(null, implementation);
183     try {
184       // If a 'get' is called on mocked interface, throw connection refused.
185       Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())).
186         thenThrow(new ServiceException(ex));
187       // Now start up the catalogtracker with our doctored Connection.
188       final CatalogTracker ct = constructAndStartCatalogTracker(connection);
189       try {
190         MetaRegionTracker.setMetaLocation(this.watcher, SN);
191         long timeout = UTIL.getConfiguration().
192           getLong("hbase.catalog.verification.timeout", 1000);
193         Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
194       } finally {
195         // Clean out meta location or later tests will be confused... they
196         // presume start fresh in zk.
197         MetaRegionTracker.deleteMetaLocation(this.watcher);
198       }
199     } finally {
200       // Clear out our doctored connection or could mess up subsequent tests.
201       HConnectionManager.deleteConnection(UTIL.getConfiguration());
202     }
203   }
204 
205   /**
206    * Test we survive a connection refused {@link ConnectException}
207    * @throws IOException
208    * @throws InterruptedException
209    * @throws KeeperException
210    * @throws ServiceException
211    */
212   @Test
213   public void testGetMetaServerConnectionFails()
214   throws IOException, InterruptedException, KeeperException, ServiceException {
215     testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
216   }
217 
218   /**
219    * Test that verifyMetaRegionLocation properly handles getting a
220    * ServerNotRunningException. See HBASE-4470.
221    * Note this doesn't check the exact exception thrown in the
222    * HBASE-4470 as there it is thrown from getHConnection() and
223    * here it is thrown from get() -- but those are both called
224    * from the same function anyway, and this way is less invasive than
225    * throwing from getHConnection would be.
226    *
227    * @throws IOException
228    * @throws InterruptedException
229    * @throws KeeperException
230    * @throws ServiceException
231    */
232   @Test
233   public void testVerifyMetaRegionServerNotRunning()
234   throws IOException, InterruptedException, KeeperException, ServiceException {
235     testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
236   }
237 
238   /**
239    * Test get of meta region fails properly if nothing to connect to.
240    * @throws IOException
241    * @throws InterruptedException
242    * @throws KeeperException
243    * @throws ServiceException
244    */
245   @Test
246   public void testVerifyMetaRegionLocationFails()
247   throws IOException, InterruptedException, KeeperException, ServiceException {
248     HConnection connection = Mockito.mock(HConnection.class);
249     ServiceException connectException =
250       new ServiceException(new ConnectException("Connection refused"));
251     final AdminProtos.AdminService.BlockingInterface implementation =
252       Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
253     Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(),
254       (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException);
255     Mockito.when(connection.getAdmin(Mockito.any(ServerName.class), Mockito.anyBoolean())).
256       thenReturn(implementation);
257     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
258     try {
259       MetaRegionTracker.setMetaLocation(this.watcher,
260           new ServerName("example.com", 1234, System.currentTimeMillis()));
261       Assert.assertFalse(ct.verifyMetaRegionLocation(100));
262     } finally {
263       // Clean out meta location or later tests will be confused... they presume
264       // start fresh in zk.
265       MetaRegionTracker.deleteMetaLocation(this.watcher);
266     }
267   }
268 
269   @Test (expected = NotAllMetaRegionsOnlineException.class)
270   public void testTimeoutWaitForMeta()
271   throws IOException, InterruptedException {
272     HConnection connection = Mockito.mock(HConnection.class);
273     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
274     ct.waitForMeta(100);
275   }
276 
277   /**
278    * Test waiting on meat w/ no timeout specified.
279    * @throws IOException
280    * @throws InterruptedException
281    * @throws KeeperException
282    */
283   @Test public void testNoTimeoutWaitForMeta()
284   throws IOException, InterruptedException, KeeperException {
285     HConnection connection = Mockito.mock(HConnection.class);
286     final CatalogTracker ct = constructAndStartCatalogTracker(connection);
287     ServerName hsa = ct.getMetaLocation();
288     Assert.assertNull(hsa);
289 
290     // Now test waiting on meta location getting set.
291     Thread t = new WaitOnMetaThread(ct);
292     startWaitAliveThenWaitItLives(t, 1);
293     // Set a meta location.
294     hsa = setMetaLocation();
295     // Join the thread... should exit shortly.
296     t.join();
297     // Now meta is available.
298     Assert.assertTrue(ct.getMetaLocation().equals(hsa));
299   }
300 
301   private ServerName setMetaLocation() throws KeeperException {
302     MetaRegionTracker.setMetaLocation(this.watcher, SN);
303     return SN;
304   }
305 
306   /**
307    * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
308    * want to pass a mocked HRS; can be null.
309    * @param client A mocked ClientProtocol instance, can be null
310    * @return Mock up a connection that returns a {@link Configuration} when
311    * {@link HConnection#getConfiguration()} is called, a 'location' when
312    * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
313    * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
314    * {@link HConnection#getAdmin(ServerName)} is called, returns the passed
315    * {@link ClientProtos.ClientService.BlockingInterface} instance when
316    * {@link HConnection#getClient(ServerName)} is called (Be sure to call
317    * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
318    * when done with this mocked Connection.
319    * @throws IOException
320    */
321   private HConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
322       final ClientProtos.ClientService.BlockingInterface client)
323   throws IOException {
324     HConnection connection =
325       HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
326     Mockito.doNothing().when(connection).close();
327     // Make it so we return any old location when asked.
328     final HRegionLocation anyLocation =
329       new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
330     Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
331         (byte[]) Mockito.any(), Mockito.anyBoolean())).
332       thenReturn(anyLocation);
333     Mockito.when(connection.locateRegion((TableName) Mockito.any(),
334         (byte[]) Mockito.any())).
335       thenReturn(anyLocation);
336     if (admin != null) {
337       // If a call to getHRegionConnection, return this implementation.
338       Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
339         thenReturn(admin);
340     }
341     if (client != null) {
342       // If a call to getClient, return this implementation.
343       Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
344         thenReturn(client);
345     }
346     return connection;
347   }
348 
349   /**
350    * @return A mocked up Result that fakes a Get on a row in the
351    * <code>.META.</code> table.
352    * @throws IOException
353    */
354   private Result getMetaTableRowResult() throws IOException {
355     return MetaMockingUtil.getMetaTableRowResult(HRegionInfo.FIRST_META_REGIONINFO, SN);
356   }
357 
358   private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
359     t.start();
360     while(!t.isAlive()) {
361       // Wait
362     }
363     // Wait one second.
364     Threads.sleep(ms);
365     Assert.assertTrue("Assert " + t.getName() + " still waiting", t.isAlive());
366   }
367 
368   class CountingProgressable implements Progressable {
369     final AtomicInteger counter = new AtomicInteger(0);
370     @Override
371     public void progress() {
372       this.counter.incrementAndGet();
373     }
374   }
375 
376   /**
377    * Wait on META.
378    */
379   class WaitOnMetaThread extends Thread {
380     final CatalogTracker ct;
381 
382     WaitOnMetaThread(final CatalogTracker ct) {
383       super("WaitOnMeta");
384       this.ct = ct;
385     }
386 
387     @Override
388     public void run() {
389       try {
390         doWaiting();
391       } catch (InterruptedException e) {
392         throw new RuntimeException("Failed wait", e);
393       }
394       LOG.info("Exiting " + getName());
395     }
396 
397     void doWaiting() throws InterruptedException {
398       try {
399         while (this.ct.waitForMeta(100) == null);
400       } catch (NotAllMetaRegionsOnlineException e) {
401         // Ignore.
402       }
403     }
404   }
405 
406 }