1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.util.ArrayList;
28  import java.util.HashMap;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Random;
32  import java.util.concurrent.SynchronousQueue;
33  import java.util.concurrent.ThreadPoolExecutor;
34  import java.util.concurrent.TimeUnit;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.hbase.HBaseConfiguration;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionLocation;
43  import org.apache.hadoop.hbase.MediumTests;
44  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
45  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
46  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.Threads;
49  import org.junit.AfterClass;
50  import org.junit.Assert;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  /**
56   * This class is for testing HCM features
57   */
58  @Category(MediumTests.class)
59  public class TestHCM {
60    private static final Log LOG = LogFactory.getLog(TestHCM.class);
61    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private static final byte[] TABLE_NAME = Bytes.toBytes("test");
63    private static final byte[] TABLE_NAME1 = Bytes.toBytes("test1");
64    private static final byte[] FAM_NAM = Bytes.toBytes("f");
65    private static final byte[] ROW = Bytes.toBytes("bbb");
66  
67    @BeforeClass
68    public static void setUpBeforeClass() throws Exception {
69      TEST_UTIL.startMiniCluster(1);
70    }
71  
72    @AfterClass public static void tearDownAfterClass() throws Exception {
73      TEST_UTIL.shutdownMiniCluster();
74    }
75  
76    /**
77     * @throws InterruptedException 
78     * @throws IllegalAccessException 
79     * @throws NoSuchFieldException 
80     * @throws ZooKeeperConnectionException 
81     * @throws IllegalArgumentException 
82     * @throws SecurityException 
83     * @see https://issues.apache.org/jira/browse/HBASE-2925
84     */
85    // Disabling.  Of course this test will OOME using new Configuration each time
86    // St.Ack 20110428
87    // @Test
88    public void testManyNewConnectionsDoesnotOOME()
89    throws SecurityException, IllegalArgumentException,
90    ZooKeeperConnectionException, NoSuchFieldException, IllegalAccessException,
91    InterruptedException {
92      createNewConfigurations();
93    }
94  
95    private static Random _randy = new Random();
96  
97    public static void createNewConfigurations() throws SecurityException,
98    IllegalArgumentException, NoSuchFieldException,
99    IllegalAccessException, InterruptedException, ZooKeeperConnectionException {
100     HConnection last = null;
101     for (int i = 0; i <= (HConnectionManager.MAX_CACHED_HBASE_INSTANCES * 2); i++) {
102       // set random key to differentiate the connection from previous ones
103       Configuration configuration = HBaseConfiguration.create();
104       configuration.set("somekey", String.valueOf(_randy.nextInt()));
105       System.out.println("Hash Code: " + configuration.hashCode());
106       HConnection connection = HConnectionManager.getConnection(configuration);
107       if (last != null) {
108         if (last == connection) {
109           System.out.println("!! Got same connection for once !!");
110         }
111       }
112       // change the configuration once, and the cached connection is lost forever:
113       //      the hashtable holding the cache won't be able to find its own keys
114       //      to remove them, so the LRU strategy does not work.
115       configuration.set("someotherkey", String.valueOf(_randy.nextInt()));
116       last = connection;
117       LOG.info("Cache Size: " + getHConnectionManagerCacheSize());
118       Thread.sleep(100);
119     }
120     Assert.assertEquals(1,
121       getHConnectionManagerCacheSize());
122   }
123 
124   private static int getHConnectionManagerCacheSize(){
125     return HConnectionTestingUtility.getConnectionCount();
126   }
127   
128   @Test
129   public void abortingHConnectionRemovesItselfFromHCM() throws Exception {
130     // Save off current HConnections
131     Map<HConnectionKey, HConnectionImplementation> oldHBaseInstances = 
132         new HashMap<HConnectionKey, HConnectionImplementation>();
133     oldHBaseInstances.putAll(HConnectionManager.HBASE_INSTANCES);
134     
135     HConnectionManager.HBASE_INSTANCES.clear();
136 
137     try {
138       HConnection connection = HConnectionManager.getConnection(TEST_UTIL.getConfiguration());
139       connection.abort("test abortingHConnectionRemovesItselfFromHCM", new Exception(
140           "test abortingHConnectionRemovesItselfFromHCM"));
141       Assert.assertNotSame(connection,
142         HConnectionManager.getConnection(TEST_UTIL.getConfiguration()));
143     } finally {
144       // Put original HConnections back
145       HConnectionManager.HBASE_INSTANCES.clear();
146       HConnectionManager.HBASE_INSTANCES.putAll(oldHBaseInstances);
147     }
148   }
149 
150   /**
151    * Test that when we delete a location using the first row of a region
152    * that we really delete it.
153    * @throws Exception
154    */
155   @Test
156   public void testRegionCaching() throws Exception{
157     HTable table = TEST_UTIL.createTable(TABLE_NAME, FAM_NAM);
158     TEST_UTIL.createMultiRegions(table, FAM_NAM);
159     Put put = new Put(ROW);
160     put.add(FAM_NAM, ROW, ROW);
161     table.put(put);
162     HConnectionManager.HConnectionImplementation conn =
163         (HConnectionManager.HConnectionImplementation)table.getConnection();
164     assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
165     conn.deleteCachedLocation(TABLE_NAME, ROW);
166     HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
167     assertNull("What is this location?? " + rl, rl);
168     table.close();
169   }
170 
171   /**
172    * Test that Connection or Pool are not closed when managed externally
173    * @throws Exception
174    */
175   @Test
176   public void testConnectionManagement() throws Exception{
177     TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM);
178     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
179     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
180         60, TimeUnit.SECONDS,
181         new SynchronousQueue<Runnable>(),
182         Threads.newDaemonThreadFactory("test-hcm-table"));
183 
184     HTable table = new HTable(TABLE_NAME1, conn, pool);
185     table.close();
186     assertFalse(conn.isClosed());
187     assertFalse(pool.isShutdown());
188     table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME1, pool);
189     table.close();
190     assertFalse(pool.isShutdown());
191     conn.close();
192     pool.shutdownNow();
193   }
194 
195   /**
196    * Make sure that {@link Configuration} instances that are essentially the
197    * same map to the same {@link HConnection} instance.
198    */
199   @Test
200   public void testConnectionSameness() throws Exception {
201     HConnection previousConnection = null;
202     for (int i = 0; i < 2; i++) {
203       // set random key to differentiate the connection from previous ones
204       Configuration configuration = TEST_UTIL.getConfiguration();
205       configuration.set("some_key", String.valueOf(_randy.nextInt()));
206       LOG.info("The hash code of the current configuration is: "
207           + configuration.hashCode());
208       HConnection currentConnection = HConnectionManager
209           .getConnection(configuration);
210       if (previousConnection != null) {
211         assertTrue(
212             "Did not get the same connection even though its key didn't change",
213             previousConnection == currentConnection);
214       }
215       previousConnection = currentConnection;
216       // change the configuration, so that it is no longer reachable from the
217       // client's perspective. However, since its part of the LRU doubly linked
218       // list, it will eventually get thrown out, at which time it should also
219       // close the corresponding {@link HConnection}.
220       configuration.set("other_key", String.valueOf(_randy.nextInt()));
221     }
222   }
223 
224   /**
225    * Makes sure that there is no leaking of
226    * {@link HConnectionManager.TableServers} in the {@link HConnectionManager}
227    * class.
228    */
229   @Test
230   public void testConnectionUniqueness() throws Exception {
231     int zkmaxconnections = TEST_UTIL.getConfiguration().
232       getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
233         HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS);
234     // Test up to a max that is < the maximum number of zk connections.  If we
235     // go above zk connections, we just fall into cycle where we are failing
236     // to set up a session and test runs for a long time.
237     int maxConnections = Math.min(zkmaxconnections - 1, 20);
238     List<HConnection> connections = new ArrayList<HConnection>(maxConnections);
239     HConnection previousConnection = null;
240     try {
241       for (int i = 0; i < maxConnections; i++) {
242         // set random key to differentiate the connection from previous ones
243         Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
244         configuration.set("some_key", String.valueOf(_randy.nextInt()));
245         configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
246             String.valueOf(_randy.nextInt()));
247         LOG.info("The hash code of the current configuration is: "
248             + configuration.hashCode());
249         HConnection currentConnection =
250           HConnectionManager.getConnection(configuration);
251         if (previousConnection != null) {
252           assertTrue("Got the same connection even though its key changed!",
253               previousConnection != currentConnection);
254         }
255         // change the configuration, so that it is no longer reachable from the
256         // client's perspective. However, since its part of the LRU doubly linked
257         // list, it will eventually get thrown out, at which time it should also
258         // close the corresponding {@link HConnection}.
259         configuration.set("other_key", String.valueOf(_randy.nextInt()));
260 
261         previousConnection = currentConnection;
262         LOG.info("The current HConnectionManager#HBASE_INSTANCES cache size is: "
263             + getHConnectionManagerCacheSize());
264         Thread.sleep(50);
265         connections.add(currentConnection);
266       }
267     } finally {
268       for (HConnection c: connections) {
269         // Clean up connections made so we don't interfere w/ subsequent tests.
270         HConnectionManager.deleteConnection(c.getConfiguration());
271       }
272     }
273   }
274 
275   @Test
276   public void testClosing() throws Exception {
277     Configuration configuration =
278       new Configuration(TEST_UTIL.getConfiguration());
279     configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
280         String.valueOf(_randy.nextInt()));
281 
282     HConnection c1 = HConnectionManager.createConnection(configuration);
283     // We create two connections with the same key.
284     HConnection c2 = HConnectionManager.createConnection(configuration);
285 
286     HConnection c3 = HConnectionManager.getConnection(configuration);
287     HConnection c4 = HConnectionManager.getConnection(configuration);
288     assertTrue(c3 == c4);
289 
290     c1.close();
291     assertTrue(c1.isClosed());
292     assertFalse(c2.isClosed());
293     assertFalse(c3.isClosed());
294 
295     c3.close();
296     // still a reference left
297     assertFalse(c3.isClosed());
298     c3.close();
299     assertTrue(c3.isClosed());
300     // c3 was removed from the cache
301     HConnection c5 = HConnectionManager.getConnection(configuration);
302     assertTrue(c5 != c3);
303 
304     assertFalse(c2.isClosed());
305     c2.close();
306     assertTrue(c2.isClosed());
307     c5.close();
308     assertTrue(c5.isClosed());
309   }
310 
311   /**
312    * Trivial test to verify that nobody messes with
313    * {@link HConnectionManager#createConnection(Configuration)}
314    */
315   @Test
316   public void testCreateConnection() throws Exception {
317     Configuration configuration = TEST_UTIL.getConfiguration();
318     HConnection c1 = HConnectionManager.createConnection(configuration);
319     HConnection c2 = HConnectionManager.createConnection(configuration);
320     // created from the same configuration, yet they are different
321     assertTrue(c1 != c2);
322     assertTrue(c1.getConfiguration() == c2.getConfiguration());
323     // make sure these were not cached
324     HConnection c3 = HConnectionManager.getConnection(configuration);
325     assertTrue(c1 != c3);
326     assertTrue(c2 != c3);
327   }
328 
329   @org.junit.Rule
330   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
331     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
332 }
333