1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.lang.reflect.Field;
29 import java.util.ArrayList;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Random;
34 import java.util.concurrent.SynchronousQueue;
35 import java.util.concurrent.ThreadPoolExecutor;
36 import java.util.concurrent.TimeUnit;
37
38 import org.apache.commons.logging.Log;
39 import org.apache.commons.logging.LogFactory;
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HRegionLocation;
45 import org.apache.hadoop.hbase.MediumTests;
46 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
47 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
48 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.Threads;
51 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
52 import org.junit.AfterClass;
53 import org.junit.Assert;
54 import org.junit.BeforeClass;
55 import org.junit.Test;
56 import org.junit.experimental.categories.Category;
57
58
59
60
61 @Category(MediumTests.class)
62 public class TestHCM {
63 private static final Log LOG = LogFactory.getLog(TestHCM.class);
64 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65 private static final byte[] TABLE_NAME = Bytes.toBytes("test");
66 private static final byte[] TABLE_NAME1 = Bytes.toBytes("test1");
67 private static final byte[] TABLE_NAME2 = Bytes.toBytes("test2");
68 private static final byte[] FAM_NAM = Bytes.toBytes("f");
69 private static final byte[] ROW = Bytes.toBytes("bbb");
70
71 @BeforeClass
72 public static void setUpBeforeClass() throws Exception {
73 TEST_UTIL.startMiniCluster(1);
74 }
75
76 @AfterClass public static void tearDownAfterClass() throws Exception {
77 TEST_UTIL.shutdownMiniCluster();
78 }
79
80
81
82
83
84
85
86
87
88
89
90
91
92 public void testManyNewConnectionsDoesnotOOME()
93 throws SecurityException, IllegalArgumentException,
94 ZooKeeperConnectionException, NoSuchFieldException, IllegalAccessException,
95 InterruptedException {
96 createNewConfigurations();
97 }
98
99 private static Random _randy = new Random();
100
101 public static void createNewConfigurations() throws SecurityException,
102 IllegalArgumentException, NoSuchFieldException,
103 IllegalAccessException, InterruptedException, ZooKeeperConnectionException {
104 HConnection last = null;
105 for (int i = 0; i <= (HConnectionManager.MAX_CACHED_HBASE_INSTANCES * 2); i++) {
106
107 Configuration configuration = HBaseConfiguration.create();
108 configuration.set("somekey", String.valueOf(_randy.nextInt()));
109 System.out.println("Hash Code: " + configuration.hashCode());
110 HConnection connection = HConnectionManager.getConnection(configuration);
111 if (last != null) {
112 if (last == connection) {
113 System.out.println("!! Got same connection for once !!");
114 }
115 }
116
117
118
119 configuration.set("someotherkey", String.valueOf(_randy.nextInt()));
120 last = connection;
121 LOG.info("Cache Size: " + getHConnectionManagerCacheSize());
122 Thread.sleep(100);
123 }
124 Assert.assertEquals(1,
125 getHConnectionManagerCacheSize());
126 }
127
128 private static int getHConnectionManagerCacheSize(){
129 return HConnectionTestingUtility.getConnectionCount();
130 }
131
132 @Test
133 public void abortingHConnectionRemovesItselfFromHCM() throws Exception {
134
135 Map<HConnectionKey, HConnectionImplementation> oldHBaseInstances =
136 new HashMap<HConnectionKey, HConnectionImplementation>();
137 oldHBaseInstances.putAll(HConnectionManager.HBASE_INSTANCES);
138
139 HConnectionManager.HBASE_INSTANCES.clear();
140
141 try {
142 HConnection connection = HConnectionManager.getConnection(TEST_UTIL.getConfiguration());
143 connection.abort("test abortingHConnectionRemovesItselfFromHCM", new Exception(
144 "test abortingHConnectionRemovesItselfFromHCM"));
145 Assert.assertNotSame(connection,
146 HConnectionManager.getConnection(TEST_UTIL.getConfiguration()));
147 } finally {
148
149 HConnectionManager.HBASE_INSTANCES.clear();
150 HConnectionManager.HBASE_INSTANCES.putAll(oldHBaseInstances);
151 }
152 }
153
154
155
156
157
158
159 @Test
160 public void testRegionCaching() throws Exception{
161 HTable table = TEST_UTIL.createTable(TABLE_NAME, FAM_NAM);
162 TEST_UTIL.createMultiRegions(table, FAM_NAM);
163 Put put = new Put(ROW);
164 put.add(FAM_NAM, ROW, ROW);
165 table.put(put);
166 HConnectionManager.HConnectionImplementation conn =
167 (HConnectionManager.HConnectionImplementation)table.getConnection();
168 assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
169 conn.deleteCachedLocation(TABLE_NAME, ROW);
170 HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
171 assertNull("What is this location?? " + rl, rl);
172 table.close();
173 }
174
175
176
177
178
179 @Test
180 public void testConnectionManagement() throws Exception{
181 TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM);
182 HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
183 ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
184 60, TimeUnit.SECONDS,
185 new SynchronousQueue<Runnable>(),
186 Threads.newDaemonThreadFactory("test-hcm-table"));
187
188 HTable table = new HTable(TABLE_NAME1, conn, pool);
189 table.close();
190 assertFalse(conn.isClosed());
191 assertFalse(pool.isShutdown());
192 table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME1, pool);
193 table.close();
194 assertFalse(pool.isShutdown());
195 conn.close();
196 pool.shutdownNow();
197 }
198
199
200
201
202
203 @Test
204 public void testConnectionSameness() throws Exception {
205 HConnection previousConnection = null;
206 for (int i = 0; i < 2; i++) {
207
208 Configuration configuration = TEST_UTIL.getConfiguration();
209 configuration.set("some_key", String.valueOf(_randy.nextInt()));
210 LOG.info("The hash code of the current configuration is: "
211 + configuration.hashCode());
212 HConnection currentConnection = HConnectionManager
213 .getConnection(configuration);
214 if (previousConnection != null) {
215 assertTrue(
216 "Did not get the same connection even though its key didn't change",
217 previousConnection == currentConnection);
218 }
219 previousConnection = currentConnection;
220
221
222
223
224 configuration.set("other_key", String.valueOf(_randy.nextInt()));
225 }
226 }
227
228
229
230
231
232
233 @Test
234 public void testConnectionUniqueness() throws Exception {
235 int zkmaxconnections = TEST_UTIL.getConfiguration().
236 getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
237 HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS);
238
239
240
241 int maxConnections = Math.min(zkmaxconnections - 1, 20);
242 List<HConnection> connections = new ArrayList<HConnection>(maxConnections);
243 HConnection previousConnection = null;
244 try {
245 for (int i = 0; i < maxConnections; i++) {
246
247 Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
248 configuration.set("some_key", String.valueOf(_randy.nextInt()));
249 configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
250 String.valueOf(_randy.nextInt()));
251 LOG.info("The hash code of the current configuration is: "
252 + configuration.hashCode());
253 HConnection currentConnection =
254 HConnectionManager.getConnection(configuration);
255 if (previousConnection != null) {
256 assertTrue("Got the same connection even though its key changed!",
257 previousConnection != currentConnection);
258 }
259
260
261
262
263 configuration.set("other_key", String.valueOf(_randy.nextInt()));
264
265 previousConnection = currentConnection;
266 LOG.info("The current HConnectionManager#HBASE_INSTANCES cache size is: "
267 + getHConnectionManagerCacheSize());
268 Thread.sleep(50);
269 connections.add(currentConnection);
270 }
271 } finally {
272 for (HConnection c: connections) {
273
274 HConnectionManager.deleteConnection(c.getConfiguration());
275 }
276 }
277 }
278
279 @Test
280 public void testClosing() throws Exception {
281 Configuration configuration =
282 new Configuration(TEST_UTIL.getConfiguration());
283 configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
284 String.valueOf(_randy.nextInt()));
285
286 HConnection c1 = HConnectionManager.createConnection(configuration);
287
288 HConnection c2 = HConnectionManager.createConnection(configuration);
289
290 HConnection c3 = HConnectionManager.getConnection(configuration);
291 HConnection c4 = HConnectionManager.getConnection(configuration);
292 assertTrue(c3 == c4);
293
294 c1.close();
295 assertTrue(c1.isClosed());
296 assertFalse(c2.isClosed());
297 assertFalse(c3.isClosed());
298
299 c3.close();
300
301 assertFalse(c3.isClosed());
302 c3.close();
303 assertTrue(c3.isClosed());
304
305 HConnection c5 = HConnectionManager.getConnection(configuration);
306 assertTrue(c5 != c3);
307
308 assertFalse(c2.isClosed());
309 c2.close();
310 assertTrue(c2.isClosed());
311 c5.close();
312 assertTrue(c5.isClosed());
313 }
314
315
316
317
318
319 @Test
320 public void testCreateConnection() throws Exception {
321 Configuration configuration = TEST_UTIL.getConfiguration();
322 HConnection c1 = HConnectionManager.createConnection(configuration);
323 HConnection c2 = HConnectionManager.createConnection(configuration);
324
325 assertTrue(c1 != c2);
326 assertTrue(c1.getConfiguration() == c2.getConfiguration());
327
328 HConnection c3 = HConnectionManager.getConnection(configuration);
329 assertTrue(c1 != c3);
330 assertTrue(c2 != c3);
331 }
332
333
334
335
336
337 @Test
338 public void testDeleteForZKConnLeak() throws Exception {
339 TEST_UTIL.createTable(TABLE_NAME2, FAM_NAM);
340 final Configuration config = TEST_UTIL.getConfiguration();
341
342 ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
343 5, TimeUnit.SECONDS,
344 new SynchronousQueue<Runnable>(),
345 Threads.newDaemonThreadFactory("test-hcm-delete"));
346
347 pool.submit(new Runnable() {
348 @Override
349 public void run() {
350 while (!Thread.interrupted()) {
351 try {
352 HConnection conn = HConnectionManager.getConnection(config);
353 HConnectionManager.deleteStaleConnection(conn);
354 } catch (Exception e) {
355 }
356 }
357 }
358 });
359
360
361 for (int i = 0; i < 10; i++) {
362 HConnection c1 = null;
363 try {
364 c1 = HConnectionManager.getConnection(config);
365 HTable table = new HTable(TABLE_NAME2, c1, pool);
366 table.close();
367 } catch (Exception e) {
368 } finally {
369 if (c1 != null) {
370 if (c1.isClosed()) {
371
372 Field zkwField = c1.getClass().getDeclaredField("zooKeeper");
373 zkwField.setAccessible(true);
374 Object watcher = zkwField.get(c1);
375
376 if (watcher != null) {
377 if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) {
378 pool.shutdownNow();
379 fail("Live zookeeper in closed connection");
380 }
381 }
382 }
383 c1.close();
384 }
385 }
386 }
387
388 pool.shutdownNow();
389 }
390
391 @org.junit.Rule
392 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
393 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
394 }
395