1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.lang.reflect.Field;
29  import java.util.ArrayList;
30  import java.util.HashMap;
31  import java.util.List;
32  import java.util.Map;
33  import java.util.Random;
34  import java.util.concurrent.SynchronousQueue;
35  import java.util.concurrent.ThreadPoolExecutor;
36  import java.util.concurrent.TimeUnit;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionLocation;
45  import org.apache.hadoop.hbase.MediumTests;
46  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
47  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
48  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.Threads;
51  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
52  import org.junit.AfterClass;
53  import org.junit.Assert;
54  import org.junit.BeforeClass;
55  import org.junit.Test;
56  import org.junit.experimental.categories.Category;
57  
58  /**
59   * This class is for testing HCM features
60   */
61  @Category(MediumTests.class)
62  public class TestHCM {
63    private static final Log LOG = LogFactory.getLog(TestHCM.class);
64    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65    private static final byte[] TABLE_NAME = Bytes.toBytes("test");
66    private static final byte[] TABLE_NAME1 = Bytes.toBytes("test1");
67    private static final byte[] TABLE_NAME2 = Bytes.toBytes("test2");
68    private static final byte[] FAM_NAM = Bytes.toBytes("f");
69    private static final byte[] ROW = Bytes.toBytes("bbb");
70  
71    @BeforeClass
72    public static void setUpBeforeClass() throws Exception {
73      TEST_UTIL.startMiniCluster(1);
74    }
75  
76    @AfterClass public static void tearDownAfterClass() throws Exception {
77      TEST_UTIL.shutdownMiniCluster();
78    }
79  
80    /**
81     * @throws InterruptedException 
82     * @throws IllegalAccessException 
83     * @throws NoSuchFieldException 
84     * @throws ZooKeeperConnectionException 
85     * @throws IllegalArgumentException 
86     * @throws SecurityException 
87     * @see https://issues.apache.org/jira/browse/HBASE-2925
88     */
89    // Disabling.  Of course this test will OOME using new Configuration each time
90    // St.Ack 20110428
91    // @Test
92    public void testManyNewConnectionsDoesnotOOME()
93    throws SecurityException, IllegalArgumentException,
94    ZooKeeperConnectionException, NoSuchFieldException, IllegalAccessException,
95    InterruptedException {
96      createNewConfigurations();
97    }
98  
99    private static Random _randy = new Random();
100 
101   public static void createNewConfigurations() throws SecurityException,
102   IllegalArgumentException, NoSuchFieldException,
103   IllegalAccessException, InterruptedException, ZooKeeperConnectionException {
104     HConnection last = null;
105     for (int i = 0; i <= (HConnectionManager.MAX_CACHED_HBASE_INSTANCES * 2); i++) {
106       // set random key to differentiate the connection from previous ones
107       Configuration configuration = HBaseConfiguration.create();
108       configuration.set("somekey", String.valueOf(_randy.nextInt()));
109       System.out.println("Hash Code: " + configuration.hashCode());
110       HConnection connection = HConnectionManager.getConnection(configuration);
111       if (last != null) {
112         if (last == connection) {
113           System.out.println("!! Got same connection for once !!");
114         }
115       }
116       // change the configuration once, and the cached connection is lost forever:
117       //      the hashtable holding the cache won't be able to find its own keys
118       //      to remove them, so the LRU strategy does not work.
119       configuration.set("someotherkey", String.valueOf(_randy.nextInt()));
120       last = connection;
121       LOG.info("Cache Size: " + getHConnectionManagerCacheSize());
122       Thread.sleep(100);
123     }
124     Assert.assertEquals(1,
125       getHConnectionManagerCacheSize());
126   }
127 
128   private static int getHConnectionManagerCacheSize(){
129     return HConnectionTestingUtility.getConnectionCount();
130   }
131   
132   @Test
133   public void abortingHConnectionRemovesItselfFromHCM() throws Exception {
134     // Save off current HConnections
135     Map<HConnectionKey, HConnectionImplementation> oldHBaseInstances = 
136         new HashMap<HConnectionKey, HConnectionImplementation>();
137     oldHBaseInstances.putAll(HConnectionManager.HBASE_INSTANCES);
138     
139     HConnectionManager.HBASE_INSTANCES.clear();
140 
141     try {
142       HConnection connection = HConnectionManager.getConnection(TEST_UTIL.getConfiguration());
143       connection.abort("test abortingHConnectionRemovesItselfFromHCM", new Exception(
144           "test abortingHConnectionRemovesItselfFromHCM"));
145       Assert.assertNotSame(connection,
146         HConnectionManager.getConnection(TEST_UTIL.getConfiguration()));
147     } finally {
148       // Put original HConnections back
149       HConnectionManager.HBASE_INSTANCES.clear();
150       HConnectionManager.HBASE_INSTANCES.putAll(oldHBaseInstances);
151     }
152   }
153 
154   /**
155    * Test that when we delete a location using the first row of a region
156    * that we really delete it.
157    * @throws Exception
158    */
159   @Test
160   public void testRegionCaching() throws Exception{
161     HTable table = TEST_UTIL.createTable(TABLE_NAME, FAM_NAM);
162     TEST_UTIL.createMultiRegions(table, FAM_NAM);
163     Put put = new Put(ROW);
164     put.add(FAM_NAM, ROW, ROW);
165     table.put(put);
166     HConnectionManager.HConnectionImplementation conn =
167         (HConnectionManager.HConnectionImplementation)table.getConnection();
168     assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
169     conn.deleteCachedLocation(TABLE_NAME, ROW);
170     HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
171     assertNull("What is this location?? " + rl, rl);
172     table.close();
173   }
174 
175   /**
176    * Test that Connection or Pool are not closed when managed externally
177    * @throws Exception
178    */
179   @Test
180   public void testConnectionManagement() throws Exception{
181     TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM);
182     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
183     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
184         60, TimeUnit.SECONDS,
185         new SynchronousQueue<Runnable>(),
186         Threads.newDaemonThreadFactory("test-hcm-table"));
187 
188     HTable table = new HTable(TABLE_NAME1, conn, pool);
189     table.close();
190     assertFalse(conn.isClosed());
191     assertFalse(pool.isShutdown());
192     table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME1, pool);
193     table.close();
194     assertFalse(pool.isShutdown());
195     conn.close();
196     pool.shutdownNow();
197   }
198 
199   /**
200    * Make sure that {@link Configuration} instances that are essentially the
201    * same map to the same {@link HConnection} instance.
202    */
203   @Test
204   public void testConnectionSameness() throws Exception {
205     HConnection previousConnection = null;
206     for (int i = 0; i < 2; i++) {
207       // set random key to differentiate the connection from previous ones
208       Configuration configuration = TEST_UTIL.getConfiguration();
209       configuration.set("some_key", String.valueOf(_randy.nextInt()));
210       LOG.info("The hash code of the current configuration is: "
211           + configuration.hashCode());
212       HConnection currentConnection = HConnectionManager
213           .getConnection(configuration);
214       if (previousConnection != null) {
215         assertTrue(
216             "Did not get the same connection even though its key didn't change",
217             previousConnection == currentConnection);
218       }
219       previousConnection = currentConnection;
220       // change the configuration, so that it is no longer reachable from the
221       // client's perspective. However, since its part of the LRU doubly linked
222       // list, it will eventually get thrown out, at which time it should also
223       // close the corresponding {@link HConnection}.
224       configuration.set("other_key", String.valueOf(_randy.nextInt()));
225     }
226   }
227 
228   /**
229    * Makes sure that there is no leaking of
230    * {@link HConnectionManager.TableServers} in the {@link HConnectionManager}
231    * class.
232    */
233   @Test
234   public void testConnectionUniqueness() throws Exception {
235     int zkmaxconnections = TEST_UTIL.getConfiguration().
236       getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
237         HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS);
238     // Test up to a max that is < the maximum number of zk connections.  If we
239     // go above zk connections, we just fall into cycle where we are failing
240     // to set up a session and test runs for a long time.
241     int maxConnections = Math.min(zkmaxconnections - 1, 20);
242     List<HConnection> connections = new ArrayList<HConnection>(maxConnections);
243     HConnection previousConnection = null;
244     try {
245       for (int i = 0; i < maxConnections; i++) {
246         // set random key to differentiate the connection from previous ones
247         Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
248         configuration.set("some_key", String.valueOf(_randy.nextInt()));
249         configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
250             String.valueOf(_randy.nextInt()));
251         LOG.info("The hash code of the current configuration is: "
252             + configuration.hashCode());
253         HConnection currentConnection =
254           HConnectionManager.getConnection(configuration);
255         if (previousConnection != null) {
256           assertTrue("Got the same connection even though its key changed!",
257               previousConnection != currentConnection);
258         }
259         // change the configuration, so that it is no longer reachable from the
260         // client's perspective. However, since its part of the LRU doubly linked
261         // list, it will eventually get thrown out, at which time it should also
262         // close the corresponding {@link HConnection}.
263         configuration.set("other_key", String.valueOf(_randy.nextInt()));
264 
265         previousConnection = currentConnection;
266         LOG.info("The current HConnectionManager#HBASE_INSTANCES cache size is: "
267             + getHConnectionManagerCacheSize());
268         Thread.sleep(50);
269         connections.add(currentConnection);
270       }
271     } finally {
272       for (HConnection c: connections) {
273         // Clean up connections made so we don't interfere w/ subsequent tests.
274         HConnectionManager.deleteConnection(c.getConfiguration());
275       }
276     }
277   }
278 
279   @Test
280   public void testClosing() throws Exception {
281     Configuration configuration =
282       new Configuration(TEST_UTIL.getConfiguration());
283     configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
284         String.valueOf(_randy.nextInt()));
285 
286     HConnection c1 = HConnectionManager.createConnection(configuration);
287     // We create two connections with the same key.
288     HConnection c2 = HConnectionManager.createConnection(configuration);
289 
290     HConnection c3 = HConnectionManager.getConnection(configuration);
291     HConnection c4 = HConnectionManager.getConnection(configuration);
292     assertTrue(c3 == c4);
293 
294     c1.close();
295     assertTrue(c1.isClosed());
296     assertFalse(c2.isClosed());
297     assertFalse(c3.isClosed());
298 
299     c3.close();
300     // still a reference left
301     assertFalse(c3.isClosed());
302     c3.close();
303     assertTrue(c3.isClosed());
304     // c3 was removed from the cache
305     HConnection c5 = HConnectionManager.getConnection(configuration);
306     assertTrue(c5 != c3);
307 
308     assertFalse(c2.isClosed());
309     c2.close();
310     assertTrue(c2.isClosed());
311     c5.close();
312     assertTrue(c5.isClosed());
313   }
314 
315   /**
316    * Trivial test to verify that nobody messes with
317    * {@link HConnectionManager#createConnection(Configuration)}
318    */
319   @Test
320   public void testCreateConnection() throws Exception {
321     Configuration configuration = TEST_UTIL.getConfiguration();
322     HConnection c1 = HConnectionManager.createConnection(configuration);
323     HConnection c2 = HConnectionManager.createConnection(configuration);
324     // created from the same configuration, yet they are different
325     assertTrue(c1 != c2);
326     assertTrue(c1.getConfiguration() == c2.getConfiguration());
327     // make sure these were not cached
328     HConnection c3 = HConnectionManager.getConnection(configuration);
329     assertTrue(c1 != c3);
330     assertTrue(c2 != c3);
331   }
332 
333   /**
334    * Tests that a destroyed connection does not have a live zookeeper.
335    * Below is timing based.  We put up a connection to a table and then close the connection while
336    * having a background thread running that is forcing close of the connection to try and
337    * provoke a close catastrophe; we are hoping for a car crash so we can see if we are leaking
338    * zk connections.
339    * @throws Exception
340    */
341   @Test
342   public void testDeleteForZKConnLeak() throws Exception {
343     TEST_UTIL.createTable(TABLE_NAME2, FAM_NAM);
344     final Configuration config = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
345     config.setInt("zookeeper.recovery.retry", 1);
346     config.setInt("zookeeper.recovery.retry.intervalmill", 1000);
347     config.setInt("hbase.rpc.timeout", 2000);
348     config.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
349 
350     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
351       5, TimeUnit.SECONDS,
352       new SynchronousQueue<Runnable>(),
353       Threads.newDaemonThreadFactory("test-hcm-delete"));
354 
355     pool.submit(new Runnable() {
356       @Override
357       public void run() {
358         while (!Thread.interrupted()) {
359           try {
360             HConnection conn = HConnectionManager.getConnection(config);
361             LOG.info("Connection " + conn);
362             HConnectionManager.deleteStaleConnection(conn);
363             LOG.info("Connection closed " + conn);
364             // TODO: This sleep time should be less than the time that it takes to open and close
365             // a table.  Ideally we would do a few runs first to measure.  For now this is
366             // timing based; hopefully we hit the bad condition.
367             Threads.sleep(10);
368           } catch (Exception e) {
369           }
370         }
371       }
372     });
373 
374     // Use connection multiple times.
375     for (int i = 0; i < 30; i++) {
376       HConnection c1 = null;
377       try {
378         c1 = HConnectionManager.getConnection(config);
379         LOG.info("HTable connection " + i + " " + c1);
380         HTable table = new HTable(TABLE_NAME2, c1, pool);
381         table.close();
382         LOG.info("HTable connection " + i + " closed " + c1);
383       } catch (Exception e) {
384         LOG.info("We actually want this to happen!!!!  So we can see if we are leaking zk", e);
385       } finally {
386         if (c1 != null) {
387           if (c1.isClosed()) {
388             // cannot use getZooKeeper as method instantiates watcher if null
389             Field zkwField = c1.getClass().getDeclaredField("zooKeeper");
390             zkwField.setAccessible(true);
391             Object watcher = zkwField.get(c1);
392 
393             if (watcher != null) {
394               if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) {
395                 // non-synchronized access to watcher; sleep and check again in case zk connection
396                 // hasn't been cleaned up yet.
397                 Thread.sleep(1000);
398                 if (((ZooKeeperWatcher) watcher).getRecoverableZooKeeper().getState().isAlive()) {
399                   pool.shutdownNow();
400                   fail("Live zookeeper in closed connection");
401                 }
402               }
403             }
404           }
405           c1.close();
406         }
407       }
408     }
409     pool.shutdownNow();
410   }
411 
412   @org.junit.Rule
413   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
414     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
415 }
416