View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.assertFalse;
25  
26  import java.io.IOException;
27  import java.net.ConnectException;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.client.ClusterConnection;
33  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
34  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
35  import org.apache.hadoop.hbase.master.RegionState;
36  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
37  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
38  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
39  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
40  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
41  import org.apache.hadoop.hbase.testclassification.MediumTests;
42  import org.apache.hadoop.hbase.util.Threads;
43  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
44  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
45  import org.apache.zookeeper.KeeperException;
46  import org.junit.After;
47  import org.junit.AfterClass;
48  import org.junit.Before;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  import org.mockito.Mockito;
53  
54  import com.google.protobuf.RpcController;
55  import com.google.protobuf.ServiceException;
56  
57  /**
58   * Test {@link org.apache.hadoop.hbase.zookeeper.MetaTableLocator}
59   */
60  @Category(MediumTests.class)
61  public class TestMetaTableLocator {
62    private static final Log LOG = LogFactory.getLog(TestMetaTableLocator.class);
63    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
64    private static final ServerName SN =
65        ServerName.valueOf("example.org", 1234, System.currentTimeMillis());
66    private ZooKeeperWatcher watcher;
67    private Abortable abortable;
68  
69    @BeforeClass public static void beforeClass() throws Exception {
70      // Set this down so tests run quicker
71      UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
72      UTIL.startMiniZKCluster();
73    }
74  
75    @AfterClass public static void afterClass() throws IOException {
76      UTIL.getZkCluster().shutdown();
77    }
78  
79    @Before public void before() throws IOException {
80      this.abortable = new Abortable() {
81        @Override
82        public void abort(String why, Throwable e) {
83          LOG.info(why, e);
84        }
85  
86        @Override
87        public boolean isAborted()  {
88          return false;
89        }
90      };
91      this.watcher = new ZooKeeperWatcher(UTIL.getConfiguration(),
92        this.getClass().getSimpleName(), this.abortable, true);
93    }
94  
95    @After public void after() {
96      try {
97        // Clean out meta location or later tests will be confused... they presume
98        // start fresh in zk.
99        new MetaTableLocator().deleteMetaLocation(this.watcher);
100     } catch (KeeperException e) {
101       LOG.warn("Unable to delete hbase:meta location", e);
102     }
103 
104     this.watcher.close();
105   }
106 
107   /**
108    * Test normal operations
109    */
110   @Test public void testMetaLookup()
111           throws IOException, InterruptedException, ServiceException, KeeperException {
112     final ClientProtos.ClientService.BlockingInterface client =
113             Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
114 
115     Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
116             thenReturn(GetResponse.newBuilder().build());
117 
118     final MetaTableLocator mtl = new MetaTableLocator();
119     assertNull(mtl.getMetaRegionLocation(this.watcher));
120     for (RegionState.State state : RegionState.State.values()) {
121       if (state.equals(RegionState.State.OPEN))
122         continue;
123       MetaTableLocator.setMetaLocation(this.watcher, SN, state);
124       assertNull(mtl.getMetaRegionLocation(this.watcher));
125       assertEquals(state, MetaTableLocator.getMetaRegionState(this.watcher).getState());
126     }
127     MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
128     assertEquals(mtl.getMetaRegionLocation(this.watcher), SN);
129     assertEquals(RegionState.State.OPEN,
130       MetaTableLocator.getMetaRegionState(this.watcher).getState());
131 
132     mtl.deleteMetaLocation(this.watcher);
133     assertNull(MetaTableLocator.getMetaRegionState(this.watcher).getServerName());
134     assertEquals(MetaTableLocator.getMetaRegionState(this.watcher).getState(),
135       RegionState.State.OFFLINE);
136     assertNull(mtl.getMetaRegionLocation(this.watcher));
137   }
138 
139 
140   /**
141    * Test interruptable while blocking wait on meta.
142    * @throws IOException
143    * @throws ServiceException
144    * @throws InterruptedException
145    */
146   @Test public void testInterruptWaitOnMeta()
147   throws IOException, InterruptedException, ServiceException {
148     final ClientProtos.ClientService.BlockingInterface client =
149       Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
150 
151     Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
152     thenReturn(GetResponse.newBuilder().build());
153 
154     final MetaTableLocator mtl = new MetaTableLocator();
155     ServerName meta = new MetaTableLocator().getMetaRegionLocation(this.watcher);
156     assertNull(meta);
157     Thread t = new Thread() {
158       @Override
159       public void run() {
160         try {
161           mtl.waitMetaRegionLocation(watcher);
162         } catch (InterruptedException e) {
163           throw new RuntimeException("Interrupted", e);
164         }
165       }
166     };
167     t.start();
168     while (!t.isAlive())
169       Threads.sleep(1);
170     Threads.sleep(1);
171     assertTrue(t.isAlive());
172     mtl.stop();
173     // Join the thread... should exit shortly.
174     t.join();
175   }
176 
177   private void testVerifyMetaRegionLocationWithException(Exception ex)
178   throws IOException, InterruptedException, KeeperException, ServiceException {
179     // Mock an ClientProtocol.
180     final ClientProtos.ClientService.BlockingInterface implementation =
181       Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
182  
183     ClusterConnection connection = mockConnection(null, implementation);
184 
185     // If a 'get' is called on mocked interface, throw connection refused.
186     Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())).
187       thenThrow(new ServiceException(ex));
188 
189     long timeout = UTIL.getConfiguration().
190             getLong("hbase.catalog.verification.timeout", 1000);
191     MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPENING);
192     assertFalse(new MetaTableLocator().verifyMetaRegionLocation(
193       connection, watcher, timeout));
194 
195     MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
196     assertFalse(new MetaTableLocator().verifyMetaRegionLocation(
197             connection, watcher, timeout));
198   }
199 
200   /**
201    * Test we survive a connection refused {@link ConnectException}
202    * @throws IOException
203    * @throws InterruptedException
204    * @throws KeeperException
205    * @throws ServiceException
206    */
207   @Test
208   public void testGetMetaServerConnectionFails()
209   throws IOException, InterruptedException, KeeperException, ServiceException {
210     testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
211   }
212 
213   /**
214    * Test that verifyMetaRegionLocation properly handles getting a
215    * ServerNotRunningException. See HBASE-4470.
216    * Note this doesn't check the exact exception thrown in the
217    * HBASE-4470 as there it is thrown from getHConnection() and
218    * here it is thrown from get() -- but those are both called
219    * from the same function anyway, and this way is less invasive than
220    * throwing from getHConnection would be.
221    *
222    * @throws IOException
223    * @throws InterruptedException
224    * @throws KeeperException
225    * @throws ServiceException
226    */
227   @Test
228   public void testVerifyMetaRegionServerNotRunning()
229   throws IOException, InterruptedException, KeeperException, ServiceException {
230     testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
231   }
232 
233   /**
234    * Test get of meta region fails properly if nothing to connect to.
235    * @throws IOException
236    * @throws InterruptedException
237    * @throws KeeperException
238    * @throws ServiceException
239    */
240   @Test
241   public void testVerifyMetaRegionLocationFails()
242   throws IOException, InterruptedException, KeeperException, ServiceException {
243     ClusterConnection connection = Mockito.mock(ClusterConnection.class);
244     ServiceException connectException =
245       new ServiceException(new ConnectException("Connection refused"));
246     final AdminProtos.AdminService.BlockingInterface implementation =
247       Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
248     Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(),
249       (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException);
250     Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
251       thenReturn(implementation);
252 
253     ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis());
254     MetaTableLocator.setMetaLocation(this.watcher,
255             sn,
256             RegionState.State.OPENING);
257     assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100));
258     MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN);
259     assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100));
260   }
261 
262   @Test (expected = NotAllMetaRegionsOnlineException.class)
263   public void testTimeoutWaitForMeta()
264   throws IOException, InterruptedException {
265     new MetaTableLocator().waitMetaRegionLocation(watcher, 100);
266   }
267 
268   /**
269    * Test waiting on meat w/ no timeout specified.
270    * @throws IOException
271    * @throws InterruptedException
272    * @throws KeeperException
273    */
274   @Test public void testNoTimeoutWaitForMeta()
275   throws IOException, InterruptedException, KeeperException {
276     final MetaTableLocator mtl = new MetaTableLocator();
277     ServerName hsa = mtl.getMetaRegionLocation(watcher);
278     assertNull(hsa);
279 
280     // Now test waiting on meta location getting set.
281     Thread t = new WaitOnMetaThread();
282     startWaitAliveThenWaitItLives(t, 1);
283     // Set a meta location.
284     MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
285     hsa = SN;
286     // Join the thread... should exit shortly.
287     t.join();
288     // Now meta is available.
289     assertTrue(mtl.getMetaRegionLocation(watcher).equals(hsa));
290   }
291 
292   /**
293    * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
294    * want to pass a mocked HRS; can be null.
295    * @param client A mocked ClientProtocol instance, can be null
296    * @return Mock up a connection that returns a {@link Configuration} when
297    * {@link HConnection#getConfiguration()} is called, a 'location' when
298    * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
299    * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
300    * {@link HConnection#getAdmin(ServerName)} is called, returns the passed
301    * {@link ClientProtos.ClientService.BlockingInterface} instance when
302    * {@link HConnection#getClient(ServerName)} is called.
303    * @throws IOException
304    */
305   private ClusterConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
306       final ClientProtos.ClientService.BlockingInterface client)
307   throws IOException {
308     ClusterConnection connection =
309       HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
310     Mockito.doNothing().when(connection).close();
311     // Make it so we return any old location when asked.
312     final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
313     Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
314         (byte[]) Mockito.any(), Mockito.anyBoolean())).
315       thenReturn(anyLocation);
316     Mockito.when(connection.locateRegion((TableName) Mockito.any(),
317         (byte[]) Mockito.any())).
318       thenReturn(anyLocation);
319     if (admin != null) {
320       // If a call to getHRegionConnection, return this implementation.
321       Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
322         thenReturn(admin);
323     }
324     if (client != null) {
325       // If a call to getClient, return this implementation.
326       Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
327         thenReturn(client);
328     }
329     return connection;
330   }
331 
332   private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
333     t.start();
334     while(!t.isAlive()) {
335       // Wait
336     }
337     // Wait one second.
338     Threads.sleep(ms);
339     assertTrue("Assert " + t.getName() + " still waiting", t.isAlive());
340   }
341 
342   /**
343    * Wait on META.
344    */
345   class WaitOnMetaThread extends Thread {
346 
347     WaitOnMetaThread() {
348       super("WaitOnMeta");
349     }
350 
351     @Override
352     public void run() {
353       try {
354         doWaiting();
355       } catch (InterruptedException e) {
356         throw new RuntimeException("Failed wait", e);
357       }
358       LOG.info("Exiting " + getName());
359     }
360 
361     void doWaiting() throws InterruptedException {
362       try {
363         while (new MetaTableLocator().waitMetaRegionLocation(watcher, 10000) == null);
364       } catch (NotAllMetaRegionsOnlineException e) {
365         //Ignore
366       }
367     }
368   }
369 }