1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.lang.reflect.Field;
30 import java.lang.reflect.Modifier;
31 import java.net.SocketTimeoutException;
32 import java.util.ArrayList;
33 import java.util.HashMap;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.Random;
37 import java.util.concurrent.ExecutorService;
38 import java.util.concurrent.SynchronousQueue;
39 import java.util.concurrent.ThreadPoolExecutor;
40 import java.util.concurrent.TimeUnit;
41 import java.util.concurrent.atomic.AtomicBoolean;
42
43 import org.apache.commons.logging.Log;
44 import org.apache.commons.logging.LogFactory;
45 import org.apache.hadoop.conf.Configuration;
46 import org.apache.hadoop.hbase.HBaseConfiguration;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HConstants;
49 import org.apache.hadoop.hbase.HRegionInfo;
50 import org.apache.hadoop.hbase.HRegionLocation;
51 import org.apache.hadoop.hbase.MediumTests;
52 import org.apache.hadoop.hbase.ServerName;
53 import org.apache.hadoop.hbase.TableName;
54 import org.apache.hadoop.hbase.Waiter;
55 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
56 import org.apache.hadoop.hbase.exceptions.DeserializationException;
57 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
58 import org.apache.hadoop.hbase.filter.Filter;
59 import org.apache.hadoop.hbase.filter.FilterBase;
60 import org.apache.hadoop.hbase.master.HMaster;
61 import org.apache.hadoop.hbase.regionserver.HRegion;
62 import org.apache.hadoop.hbase.regionserver.HRegionServer;
63 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
64 import org.apache.hadoop.hbase.util.Bytes;
65 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
66 import org.apache.hadoop.hbase.util.JVMClusterUtil;
67 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
68 import org.apache.hadoop.hbase.util.Threads;
69 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
70 import org.junit.AfterClass;
71 import org.junit.Assert;
72 import org.junit.BeforeClass;
73 import org.junit.Ignore;
74 import org.junit.Test;
75 import org.junit.experimental.categories.Category;
76
77 import com.google.common.collect.Lists;
78
79
80
81
82 @Category(MediumTests.class)
83 public class TestHCM {
84 private static final Log LOG = LogFactory.getLog(TestHCM.class);
85 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
86 private static final TableName TABLE_NAME =
87 TableName.valueOf("test");
88 private static final TableName TABLE_NAME1 =
89 TableName.valueOf("test1");
90 private static final TableName TABLE_NAME2 =
91 TableName.valueOf("test2");
92 private static final TableName TABLE_NAME3 =
93 TableName.valueOf("test3");
94 private static final TableName TABLE_NAME4 =
95 TableName.valueOf("test4");
96 private static final byte[] FAM_NAM = Bytes.toBytes("f");
97 private static final byte[] ROW = Bytes.toBytes("bbb");
98 private static final byte[] ROW_X = Bytes.toBytes("xxx");
99 private static Random _randy = new Random();
100
101 @BeforeClass
102 public static void setUpBeforeClass() throws Exception {
103 TEST_UTIL.getConfiguration().setBoolean(HConstants.STATUS_PUBLISHED, true);
104 TEST_UTIL.startMiniCluster(2);
105 }
106
107 @AfterClass public static void tearDownAfterClass() throws Exception {
108 TEST_UTIL.shutdownMiniCluster();
109 }
110
111
112 private static int getHConnectionManagerCacheSize(){
113 return HConnectionTestingUtility.getConnectionCount();
114 }
115
116 @Test
117 public void testClusterConnection() throws IOException {
118 ThreadPoolExecutor otherPool = new ThreadPoolExecutor(1, 1,
119 5, TimeUnit.SECONDS,
120 new SynchronousQueue<Runnable>(),
121 Threads.newDaemonThreadFactory("test-hcm"));
122
123 HConnection con1 = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
124 HConnection con2 = HConnectionManager.createConnection(TEST_UTIL.getConfiguration(), otherPool);
125
126 assertTrue(otherPool == ((HConnectionImplementation)con2).getCurrentBatchPool());
127
128 String tableName = "testClusterConnection";
129 TEST_UTIL.createTable(tableName.getBytes(), FAM_NAM).close();
130 HTable t = (HTable)con1.getTable(tableName, otherPool);
131
132 assertNull("Internal Thread pool should be null", ((HConnectionImplementation)con1).getCurrentBatchPool());
133
134 assertTrue(otherPool == t.getPool());
135 t.close();
136
137 t = (HTable)con2.getTable(tableName);
138
139 assertTrue(otherPool == t.getPool());
140 t.close();
141
142 t = (HTable)con2.getTable(Bytes.toBytes(tableName));
143
144 assertTrue(otherPool == t.getPool());
145 t.close();
146
147 t = (HTable)con2.getTable(TableName.valueOf(tableName));
148
149 assertTrue(otherPool == t.getPool());
150 t.close();
151
152 t = (HTable)con1.getTable(tableName);
153 ExecutorService pool = ((HConnectionImplementation)con1).getCurrentBatchPool();
154
155 assertNotNull("An internal Thread pool should have been created", pool);
156
157 assertTrue(t.getPool() == pool);
158 t.close();
159
160 t = (HTable)con1.getTable(tableName);
161
162 assertTrue(t.getPool() == pool);
163 t.close();
164
165 con1.close();
166
167 assertTrue(pool.isShutdown());
168
169 con2.close();
170
171 assertFalse(otherPool.isShutdown());
172 otherPool.shutdownNow();
173 }
174
175 @Ignore ("Fails in IDEs: HBASE-9042") @Test(expected = RegionServerStoppedException.class)
176 public void testClusterStatus() throws Exception {
177 TableName tn =
178 TableName.valueOf("testClusterStatus");
179 byte[] cf = "cf".getBytes();
180 byte[] rk = "rk1".getBytes();
181
182 JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
183 rs.waitForServerOnline();
184 final ServerName sn = rs.getRegionServer().getServerName();
185
186 HTable t = TEST_UTIL.createTable(tn, cf);
187 TEST_UTIL.waitTableAvailable(tn.getName());
188
189 while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
190 getRegionStates().isRegionsInTransition()){
191 Thread.sleep(1);
192 }
193 final HConnectionImplementation hci = (HConnectionImplementation)t.getConnection();
194 while (t.getRegionLocation(rk).getPort() != sn.getPort()){
195 TEST_UTIL.getHBaseAdmin().move(t.getRegionLocation(rk).getRegionInfo().
196 getEncodedNameAsBytes(), Bytes.toBytes(sn.toString()));
197 while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
198 getRegionStates().isRegionsInTransition()){
199 Thread.sleep(1);
200 }
201 hci.clearRegionCache(tn);
202 }
203 Assert.assertNotNull(hci.clusterStatusListener);
204 TEST_UTIL.assertRegionOnServer(t.getRegionLocation(rk).getRegionInfo(), sn, 20000);
205
206 Put p1 = new Put(rk);
207 p1.add(cf, "qual".getBytes(), "val".getBytes());
208 t.put(p1);
209
210 rs.getRegionServer().abort("I'm dead");
211
212
213 TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() {
214 @Override
215 public boolean evaluate() throws Exception {
216 return TEST_UTIL.getHBaseCluster().getMaster().getServerManager().
217 getDeadServers().isDeadServer(sn);
218 }
219 });
220
221 TEST_UTIL.waitFor(40000, 1000, true, new Waiter.Predicate<Exception>() {
222 @Override
223 public boolean evaluate() throws Exception {
224 return hci.clusterStatusListener.isDeadServer(sn);
225 }
226 });
227
228 hci.getClient(sn);
229 }
230
231
232
233
234
235
236 @Test
237 public void testConnectionCut() throws Exception {
238 String tableName = "HCM-testConnectionCut";
239
240 TEST_UTIL.createTable(tableName.getBytes(), FAM_NAM).close();
241 boolean previousBalance = TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
242
243 Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
244
245 c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
246 c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
247 c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 30 * 1000);
248
249 HTable table = new HTable(c2, tableName);
250
251 Put p = new Put(FAM_NAM);
252 p.add(FAM_NAM, FAM_NAM, FAM_NAM);
253 table.put(p);
254
255 final HConnectionImplementation hci = (HConnectionImplementation)table.getConnection();
256 final HRegionLocation loc = table.getRegionLocation(FAM_NAM);
257
258 Get get = new Get(FAM_NAM);
259 Assert.assertNotNull(table.get(get));
260
261 get = new Get(FAM_NAM);
262 get.setFilter(new BlockingFilter());
263
264
265 Thread t = new Thread() {
266 @Override
267 public void run() {
268 synchronized (syncBlockingFilter) {
269 try {
270 syncBlockingFilter.wait();
271 } catch (InterruptedException e) {
272 throw new RuntimeException(e);
273 }
274 }
275 hci.clusterStatusListener.deadServerHandler.newDead(loc.getServerName());
276 }
277 };
278
279 t.start();
280 try {
281 table.get(get);
282 Assert.fail();
283 } catch (IOException expected) {
284 LOG.debug("Received: " + expected);
285 Assert.assertFalse(expected instanceof SocketTimeoutException);
286 Assert.assertFalse(syncBlockingFilter.get());
287 } finally {
288 syncBlockingFilter.set(true);
289 t.join();
290 HConnectionManager.getConnection(c2).close();
291 TEST_UTIL.getHBaseAdmin().setBalancerRunning(previousBalance, true);
292 }
293
294 table.close();
295 }
296
297 protected static final AtomicBoolean syncBlockingFilter = new AtomicBoolean(false);
298
299 public static class BlockingFilter extends FilterBase {
300 @Override
301 public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
302 int i = 0;
303 while (i++ < 1000 && !syncBlockingFilter.get()) {
304 synchronized (syncBlockingFilter) {
305 syncBlockingFilter.notifyAll();
306 }
307 Threads.sleep(100);
308 }
309 syncBlockingFilter.set(true);
310 return false;
311 }
312
313 public static Filter parseFrom(final byte [] pbBytes) throws DeserializationException{
314 return new BlockingFilter();
315 }
316 }
317
318 @Test
319 public void abortingHConnectionRemovesItselfFromHCM() throws Exception {
320
321 Map<HConnectionKey, HConnectionImplementation> oldHBaseInstances =
322 new HashMap<HConnectionKey, HConnectionImplementation>();
323 oldHBaseInstances.putAll(HConnectionManager.CONNECTION_INSTANCES);
324
325 HConnectionManager.CONNECTION_INSTANCES.clear();
326
327 try {
328 HConnection connection = HConnectionManager.getConnection(TEST_UTIL.getConfiguration());
329 connection.abort("test abortingHConnectionRemovesItselfFromHCM", new Exception(
330 "test abortingHConnectionRemovesItselfFromHCM"));
331 Assert.assertNotSame(connection,
332 HConnectionManager.getConnection(TEST_UTIL.getConfiguration()));
333 } finally {
334
335 HConnectionManager.CONNECTION_INSTANCES.clear();
336 HConnectionManager.CONNECTION_INSTANCES.putAll(oldHBaseInstances);
337 }
338 }
339
340
341
342
343
344
345 @Test
346 public void testRegionCaching() throws Exception{
347 TEST_UTIL.createTable(TABLE_NAME, FAM_NAM).close();
348 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
349 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
350 HTable table = new HTable(conf, TABLE_NAME);
351
352 TEST_UTIL.createMultiRegions(table, FAM_NAM);
353 TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
354 Put put = new Put(ROW);
355 put.add(FAM_NAM, ROW, ROW);
356 table.put(put);
357 HConnectionManager.HConnectionImplementation conn =
358 (HConnectionManager.HConnectionImplementation)table.getConnection();
359
360 assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
361
362 final int nextPort = conn.getCachedLocation(TABLE_NAME, ROW).getPort() + 1;
363 HRegionLocation loc = conn.getCachedLocation(TABLE_NAME, ROW);
364 conn.updateCachedLocation(loc.getRegionInfo(), loc, ServerName.valueOf("127.0.0.1", nextPort,
365 HConstants.LATEST_TIMESTAMP), HConstants.LATEST_TIMESTAMP);
366 Assert.assertEquals(conn.getCachedLocation(TABLE_NAME, ROW).getPort(), nextPort);
367
368 conn.forceDeleteCachedLocation(TABLE_NAME, ROW.clone());
369 HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
370 assertNull("What is this location?? " + rl, rl);
371
372
373
374 conn.clearRegionCache(TABLE_NAME);
375 Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME));
376 Put put2 = new Put(ROW);
377 put2.add(FAM_NAM, ROW, ROW);
378 table.put(put2);
379 assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
380 assertNotNull(conn.getCachedLocation(TableName.valueOf(TABLE_NAME.getName()), ROW.clone()));
381
382 TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, false);
383 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
384
385
386 while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
387 Thread.sleep(1);
388 }
389
390
391 HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME, ROW);
392 byte[] regionName = toMove.getRegionInfo().getRegionName();
393 byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
394
395
396 int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
397 int destServerId = (curServerId == 0 ? 1 : 0);
398
399 HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
400 HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
401
402 ServerName destServerName = destServer.getServerName();
403
404
405 Assert.assertTrue(curServer != destServer);
406 Assert.assertFalse(curServer.getServerName().equals(destServer.getServerName()));
407 Assert.assertFalse( toMove.getPort() == destServerName.getPort());
408 Assert.assertNotNull(curServer.getOnlineRegion(regionName));
409 Assert.assertNull(destServer.getOnlineRegion(regionName));
410 Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().
411 getAssignmentManager().getRegionStates().isRegionsInTransition());
412
413
414
415 LOG.info("Move starting region="+toMove.getRegionInfo().getRegionNameAsString());
416 TEST_UTIL.getHBaseAdmin().move(
417 toMove.getRegionInfo().getEncodedNameAsBytes(),
418 destServerName.getServerName().getBytes()
419 );
420
421 while (destServer.getOnlineRegion(regionName) == null ||
422 destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
423 curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
424 master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
425
426 Thread.sleep(1);
427 }
428
429 LOG.info("Move finished for region="+toMove.getRegionInfo().getRegionNameAsString());
430
431
432 Assert.assertNull(curServer.getOnlineRegion(regionName));
433 Assert.assertNotNull(destServer.getOnlineRegion(regionName));
434 Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
435 Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
436
437
438
439 Assert.assertFalse(
440 conn.getCachedLocation(TABLE_NAME, ROW).getPort() == destServerName.getPort());
441
442
443
444 LOG.info("Put starting");
445 Put put3 = new Put(ROW);
446 put3.add(FAM_NAM, ROW, ROW);
447 try {
448 table.put(put3);
449 Assert.fail("Unreachable point");
450 } catch (RetriesExhaustedWithDetailsException e){
451 LOG.info("Put done, exception caught: " + e.getClass());
452 Assert.assertEquals(1, e.getNumExceptions());
453 Assert.assertEquals(1, e.getCauses().size());
454 Assert.assertArrayEquals(e.getRow(0).getRow(), ROW);
455
456
457 Throwable cause = HConnectionManager.findException(e.getCause(0));
458 Assert.assertNotNull(cause);
459 Assert.assertTrue(cause instanceof RegionMovedException);
460 }
461 Assert.assertNotNull("Cached connection is null", conn.getCachedLocation(TABLE_NAME, ROW));
462 Assert.assertEquals(
463 "Previous server was " + curServer.getServerName().getHostAndPort(),
464 destServerName.getPort(), conn.getCachedLocation(TABLE_NAME, ROW).getPort());
465
466 Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
467 Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
468
469
470 LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
471 TEST_UTIL.getHBaseAdmin().move(
472 toMove.getRegionInfo().getEncodedNameAsBytes(),
473 curServer.getServerName().getServerName().getBytes()
474 );
475
476 while (curServer.getOnlineRegion(regionName) == null ||
477 destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
478 curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
479 master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
480
481 Thread.sleep(1);
482 }
483
484
485 Assert.assertNotNull(curServer.getOnlineRegion(regionName));
486 Assert.assertNull(destServer.getOnlineRegion(regionName));
487 LOG.info("Move finished for region=" + toMove.getRegionInfo().getRegionNameAsString());
488
489
490 Assert.assertFalse(conn.getCachedLocation(TABLE_NAME, ROW).getPort() ==
491 curServer.getServerName().getPort());
492
493 Scan sc = new Scan();
494 sc.setStopRow(ROW);
495 sc.setStartRow(ROW);
496
497
498
499 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
500
501 try {
502 ResultScanner rs = table.getScanner(sc);
503 while (rs.next() != null) {
504 }
505 Assert.fail("Unreachable point");
506 } catch (RetriesExhaustedException e) {
507 LOG.info("Scan done, expected exception caught: " + e.getClass());
508 }
509
510
511 Assert.assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
512 Assert.assertEquals(
513 "Previous server was "+destServer.getServerName().getHostAndPort(),
514 curServer.getServerName().getPort(), conn.getCachedLocation(TABLE_NAME, ROW).getPort());
515
516 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
517 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
518 table.close();
519 }
520
521
522
523
524
525 @Test
526 public void testConnectionManagement() throws Exception{
527 TEST_UTIL.createTable(TABLE_NAME1, FAM_NAM);
528 HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
529 HTableInterface table = conn.getTable(TABLE_NAME1.getName());
530 table.close();
531 assertFalse(conn.isClosed());
532 assertFalse(((HTable)table).getPool().isShutdown());
533 table = conn.getTable(TABLE_NAME1.getName());
534 table.close();
535 assertFalse(((HTable)table).getPool().isShutdown());
536 conn.close();
537 assertTrue(((HTable)table).getPool().isShutdown());
538 }
539
540
541
542
543 @Test(timeout = 60000)
544 public void testCacheSeqNums() throws Exception{
545 HTable table = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAM);
546 TEST_UTIL.createMultiRegions(table, FAM_NAM);
547 Put put = new Put(ROW);
548 put.add(FAM_NAM, ROW, ROW);
549 table.put(put);
550 HConnectionManager.HConnectionImplementation conn =
551 (HConnectionManager.HConnectionImplementation)table.getConnection();
552
553 HRegionLocation location = conn.getCachedLocation(TABLE_NAME2, ROW);
554 assertNotNull(location);
555
556 HRegionLocation anySource = new HRegionLocation(location.getRegionInfo(), ServerName.valueOf(
557 location.getHostname(), location.getPort() - 1, 0L));
558
559
560 int nextPort = location.getPort() + 1;
561 conn.updateCachedLocation(location.getRegionInfo(), location,
562 ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
563 location = conn.getCachedLocation(TABLE_NAME2, ROW);
564 Assert.assertEquals(nextPort, location.getPort());
565
566
567 nextPort = location.getPort() + 1;
568 conn.updateCachedLocation(location.getRegionInfo(), location,
569 ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
570 location = conn.getCachedLocation(TABLE_NAME2, ROW);
571 Assert.assertEquals(nextPort, location.getPort());
572
573
574 nextPort = location.getPort() + 1;
575 conn.updateCachedLocation(location.getRegionInfo(), anySource,
576 ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() + 1);
577 location = conn.getCachedLocation(TABLE_NAME2, ROW);
578 Assert.assertEquals(nextPort, location.getPort());
579
580
581 nextPort = location.getPort() + 1;
582 conn.updateCachedLocation(location.getRegionInfo(), anySource,
583 ServerName.valueOf("127.0.0.1", nextPort, 0), location.getSeqNum() - 1);
584 location = conn.getCachedLocation(TABLE_NAME2, ROW);
585 Assert.assertEquals(nextPort - 1, location.getPort());
586 }
587
588
589
590
591
592 @Test
593 public void testConnectionSameness() throws Exception {
594 HConnection previousConnection = null;
595 for (int i = 0; i < 2; i++) {
596
597 Configuration configuration = TEST_UTIL.getConfiguration();
598 configuration.set("some_key", String.valueOf(_randy.nextInt()));
599 LOG.info("The hash code of the current configuration is: "
600 + configuration.hashCode());
601 HConnection currentConnection = HConnectionManager
602 .getConnection(configuration);
603 if (previousConnection != null) {
604 assertTrue(
605 "Did not get the same connection even though its key didn't change",
606 previousConnection == currentConnection);
607 }
608 previousConnection = currentConnection;
609
610
611
612
613 configuration.set("other_key", String.valueOf(_randy.nextInt()));
614 }
615 }
616
617
618
619
620
621
622 @Test
623 public void testConnectionUniqueness() throws Exception {
624 int zkmaxconnections = TEST_UTIL.getConfiguration().
625 getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
626 HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS);
627
628
629
630 int maxConnections = Math.min(zkmaxconnections - 1, 20);
631 List<HConnection> connections = new ArrayList<HConnection>(maxConnections);
632 HConnection previousConnection = null;
633 try {
634 for (int i = 0; i < maxConnections; i++) {
635
636 Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
637 configuration.set("some_key", String.valueOf(_randy.nextInt()));
638 configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
639 String.valueOf(_randy.nextInt()));
640 LOG.info("The hash code of the current configuration is: "
641 + configuration.hashCode());
642 HConnection currentConnection =
643 HConnectionManager.getConnection(configuration);
644 if (previousConnection != null) {
645 assertTrue("Got the same connection even though its key changed!",
646 previousConnection != currentConnection);
647 }
648
649
650
651
652 configuration.set("other_key", String.valueOf(_randy.nextInt()));
653
654 previousConnection = currentConnection;
655 LOG.info("The current HConnectionManager#HBASE_INSTANCES cache size is: "
656 + getHConnectionManagerCacheSize());
657 Thread.sleep(50);
658 connections.add(currentConnection);
659 }
660 } finally {
661 for (HConnection c: connections) {
662
663 HConnectionManager.deleteConnection(c.getConfiguration());
664 }
665 }
666 }
667
668 @Test
669 public void testClosing() throws Exception {
670 Configuration configuration =
671 new Configuration(TEST_UTIL.getConfiguration());
672 configuration.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
673 String.valueOf(_randy.nextInt()));
674
675 HConnection c1 = HConnectionManager.createConnection(configuration);
676
677 HConnection c2 = HConnectionManager.createConnection(configuration);
678
679 HConnection c3 = HConnectionManager.getConnection(configuration);
680 HConnection c4 = HConnectionManager.getConnection(configuration);
681 assertTrue(c3 == c4);
682
683 c1.close();
684 assertTrue(c1.isClosed());
685 assertFalse(c2.isClosed());
686 assertFalse(c3.isClosed());
687
688 c3.close();
689
690 assertFalse(c3.isClosed());
691 c3.close();
692 assertTrue(c3.isClosed());
693
694 HConnection c5 = HConnectionManager.getConnection(configuration);
695 assertTrue(c5 != c3);
696
697 assertFalse(c2.isClosed());
698 c2.close();
699 assertTrue(c2.isClosed());
700 c5.close();
701 assertTrue(c5.isClosed());
702 }
703
704
705
706
707
708 @Test
709 public void testCreateConnection() throws Exception {
710 Configuration configuration = TEST_UTIL.getConfiguration();
711 HConnection c1 = HConnectionManager.createConnection(configuration);
712 HConnection c2 = HConnectionManager.createConnection(configuration);
713
714 assertTrue(c1 != c2);
715 assertTrue(c1.getConfiguration() == c2.getConfiguration());
716
717 HConnection c3 = HConnectionManager.getConnection(configuration);
718 assertTrue(c1 != c3);
719 assertTrue(c2 != c3);
720 }
721
722
723
724
725
726
727
728 @Test(timeout = 60000)
729 public void testConnection() throws Exception{
730
731 Configuration c = new Configuration();
732 c.set(HConstants.ZOOKEEPER_QUORUM,
733 TEST_UTIL.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM));
734 c.set(HConstants.ZOOKEEPER_CLIENT_PORT ,
735 TEST_UTIL.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT));
736
737
738 HConnection conn = HConnectionManager.getConnection(c);
739 assertTrue( conn.isMasterRunning() );
740 conn.close();
741 }
742
743 private int setNumTries(HConnectionImplementation hci, int newVal) throws Exception {
744 Field numTries = hci.getClass().getDeclaredField("numTries");
745 numTries.setAccessible(true);
746 Field modifiersField = Field.class.getDeclaredField("modifiers");
747 modifiersField.setAccessible(true);
748 modifiersField.setInt(numTries, numTries.getModifiers() & ~Modifier.FINAL);
749 final int prevNumRetriesVal = (Integer)numTries.get(hci);
750 numTries.set(hci, newVal);
751
752 return prevNumRetriesVal;
753 }
754
755 @Test
756 public void testMulti() throws Exception {
757 HTable table = TEST_UTIL.createTable(TABLE_NAME3, FAM_NAM);
758 TEST_UTIL.createMultiRegions(table, FAM_NAM);
759 HConnectionManager.HConnectionImplementation conn =
760 (HConnectionManager.HConnectionImplementation)
761 HConnectionManager.getConnection(TEST_UTIL.getConfiguration());
762
763
764
765 conn.clearRegionCache(TABLE_NAME3);
766 Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME3));
767
768 TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, false);
769 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
770
771
772 while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
773 Thread.sleep(1);
774 }
775
776 Put put = new Put(ROW_X);
777 put.add(FAM_NAM, ROW_X, ROW_X);
778 table.put(put);
779
780
781 HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME3, ROW_X);
782 byte[] regionName = toMove.getRegionInfo().getRegionName();
783 byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
784
785
786 int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
787 int destServerId = (curServerId == 0 ? 1 : 0);
788
789 HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
790 HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
791
792 ServerName destServerName = destServer.getServerName();
793
794
795 List<HRegion> regions = curServer.getOnlineRegions(TABLE_NAME3);
796 byte[] otherRow = null;
797 for (HRegion region : regions) {
798 if (!region.getRegionInfo().getEncodedName().equals(toMove.getRegionInfo().getEncodedName())
799 && Bytes.BYTES_COMPARATOR.compare(region.getRegionInfo().getStartKey(), ROW_X) < 0) {
800 otherRow = region.getRegionInfo().getStartKey();
801 break;
802 }
803 }
804 assertNotNull(otherRow);
805
806 if (otherRow.length <= 0) otherRow = Bytes.toBytes("aaa");
807 Put put2 = new Put(otherRow);
808 put2.add(FAM_NAM, otherRow, otherRow);
809 table.put(put2);
810
811
812 Assert.assertTrue(curServer != destServer);
813 Assert.assertNotEquals(curServer.getServerName(), destServer.getServerName());
814 Assert.assertNotEquals(toMove.getPort(), destServerName.getPort());
815 Assert.assertNotNull(curServer.getOnlineRegion(regionName));
816 Assert.assertNull(destServer.getOnlineRegion(regionName));
817 Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().
818 getAssignmentManager().getRegionStates().isRegionsInTransition());
819
820
821
822 LOG.info("Move starting region="+toMove.getRegionInfo().getRegionNameAsString());
823 TEST_UTIL.getHBaseAdmin().move(
824 toMove.getRegionInfo().getEncodedNameAsBytes(),
825 destServerName.getServerName().getBytes()
826 );
827
828 while (destServer.getOnlineRegion(regionName) == null ||
829 destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
830 curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) ||
831 master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
832
833 Thread.sleep(1);
834 }
835
836 LOG.info("Move finished for region="+toMove.getRegionInfo().getRegionNameAsString());
837
838
839 Assert.assertNull(curServer.getOnlineRegion(regionName));
840 Assert.assertNotNull(destServer.getOnlineRegion(regionName));
841 Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
842 Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
843
844
845
846 Assert.assertFalse(
847 conn.getCachedLocation(TABLE_NAME3, ROW_X).getPort() == destServerName.getPort());
848
849
850 final int prevNumRetriesVal = setNumTries(conn, 2);
851
852 Put put3 = new Put(ROW_X);
853 put3.add(FAM_NAM, ROW_X, ROW_X);
854 Put put4 = new Put(otherRow);
855 put4.add(FAM_NAM, otherRow, otherRow);
856
857
858 table.batch(Lists.newArrayList(put4, put3));
859
860
861 setNumTries(conn, prevNumRetriesVal);
862 table.close();
863 conn.close();
864 }
865
866 @Ignore ("Test presumes RETRY_BACKOFF will never change; it has") @Test
867 public void testErrorBackoffTimeCalculation() throws Exception {
868
869 final long ANY_PAUSE = 100;
870 HRegionInfo ri = new HRegionInfo(TABLE_NAME);
871 HRegionLocation location = new HRegionLocation(ri, ServerName.valueOf("127.0.0.1", 1, 0));
872 HRegionLocation diffLocation = new HRegionLocation(ri, ServerName.valueOf("127.0.0.1", 2, 0));
873
874 ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
875 EnvironmentEdgeManager.injectEdge(timeMachine);
876 try {
877 long timeBase = timeMachine.currentTimeMillis();
878 long largeAmountOfTime = ANY_PAUSE * 1000;
879 HConnectionManager.ServerErrorTracker tracker =
880 new HConnectionManager.ServerErrorTracker(largeAmountOfTime, 100);
881
882
883 assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));
884
885
886 tracker.reportServerError(location);
887 assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(location, ANY_PAUSE));
888 tracker.reportServerError(location);
889 tracker.reportServerError(location);
890 tracker.reportServerError(location);
891 assertEqualsWithJitter(ANY_PAUSE * 5, tracker.calculateBackoffTime(location, ANY_PAUSE));
892
893
894
895 assertEquals(0, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));
896 tracker.reportServerError(diffLocation);
897 assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));
898
899
900 HRegionInfo ri2 = new HRegionInfo(TABLE_NAME2);
901 HRegionLocation diffRegion = new HRegionLocation(ri2, location.getServerName());
902 assertEqualsWithJitter(ANY_PAUSE * 5, tracker.calculateBackoffTime(diffRegion, ANY_PAUSE));
903
904
905 assertEqualsWithJitter(ANY_PAUSE * 10,
906 tracker.calculateBackoffTime(location, ANY_PAUSE * 2));
907
908
909
910 long timeShift = (long)(ANY_PAUSE * 0.5);
911 timeMachine.setValue(timeBase + timeShift);
912 assertEqualsWithJitter((ANY_PAUSE * 5) - timeShift,
913 tracker.calculateBackoffTime(location, ANY_PAUSE), ANY_PAUSE * 2);
914
915
916 timeMachine.setValue(timeBase + ANY_PAUSE * 100);
917 assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));
918
919
920 long timeLeft = (long)(ANY_PAUSE * 0.5);
921 timeMachine.setValue(timeBase + largeAmountOfTime - timeLeft);
922 assertTrue(tracker.canRetryMore(1));
923 tracker.reportServerError(location);
924 assertEquals(timeLeft, tracker.calculateBackoffTime(location, ANY_PAUSE));
925 timeMachine.setValue(timeBase + largeAmountOfTime);
926 assertFalse(tracker.canRetryMore(1));
927 } finally {
928 EnvironmentEdgeManager.reset();
929 }
930 }
931
932 private static void assertEqualsWithJitter(long expected, long actual) {
933 assertEqualsWithJitter(expected, actual, expected);
934 }
935
936 private static void assertEqualsWithJitter(long expected, long actual, long jitterBase) {
937 assertTrue("Value not within jitter: " + expected + " vs " + actual,
938 Math.abs(actual - expected) <= (0.01f * jitterBase));
939 }
940
941
942
943
944
945
946
947
948
949 @Ignore ("Flakey test: See HBASE-8996")@Test
950 public void testDeleteForZKConnLeak() throws Exception {
951 TEST_UTIL.createTable(TABLE_NAME4, FAM_NAM);
952 final Configuration config = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
953 config.setInt("zookeeper.recovery.retry", 1);
954 config.setInt("zookeeper.recovery.retry.intervalmill", 1000);
955 config.setInt("hbase.rpc.timeout", 2000);
956 config.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
957
958 ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 10,
959 5, TimeUnit.SECONDS,
960 new SynchronousQueue<Runnable>(),
961 Threads.newDaemonThreadFactory("test-hcm-delete"));
962
963 pool.submit(new Runnable() {
964 @Override
965 public void run() {
966 while (!Thread.interrupted()) {
967 try {
968 HConnection conn = HConnectionManager.getConnection(config);
969 LOG.info("Connection " + conn);
970 HConnectionManager.deleteStaleConnection(conn);
971 LOG.info("Connection closed " + conn);
972
973
974
975 Threads.sleep(10);
976 } catch (Exception e) {
977 }
978 }
979 }
980 });
981
982
983 for (int i = 0; i < 30; i++) {
984 HConnection c1 = null;
985 try {
986 c1 = HConnectionManager.getConnection(config);
987 LOG.info("HTable connection " + i + " " + c1);
988 HTable table = new HTable(TABLE_NAME4, c1, pool);
989 table.close();
990 LOG.info("HTable connection " + i + " closed " + c1);
991 } catch (Exception e) {
992 LOG.info("We actually want this to happen!!!! So we can see if we are leaking zk", e);
993 } finally {
994 if (c1 != null) {
995 if (c1.isClosed()) {
996
997 Field zkwField = c1.getClass().getDeclaredField("keepAliveZookeeper");
998 zkwField.setAccessible(true);
999 Object watcher = zkwField.get(c1);
1000
1001 if (watcher != null) {
1002 if (((ZooKeeperWatcher)watcher).getRecoverableZooKeeper().getState().isAlive()) {
1003
1004
1005 Thread.sleep(1000);
1006 if (((ZooKeeperWatcher) watcher).getRecoverableZooKeeper().getState().isAlive()) {
1007 pool.shutdownNow();
1008 fail("Live zookeeper in closed connection");
1009 }
1010 }
1011 }
1012 }
1013 c1.close();
1014 }
1015 }
1016 }
1017 pool.shutdownNow();
1018 }
1019 }
1020