1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.io.Closeable;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.lang.reflect.Constructor;
25 import java.lang.reflect.UndeclaredThrowableException;
26 import java.net.SocketException;
27 import java.util.ArrayList;
28 import java.util.Date;
29 import java.util.HashSet;
30 import java.util.LinkedHashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Map.Entry;
34 import java.util.NavigableMap;
35 import java.util.Set;
36 import java.util.concurrent.ConcurrentHashMap;
37 import java.util.concurrent.ConcurrentMap;
38 import java.util.concurrent.ConcurrentSkipListMap;
39 import java.util.concurrent.ConcurrentSkipListSet;
40 import java.util.concurrent.CopyOnWriteArraySet;
41 import java.util.concurrent.ExecutorService;
42 import java.util.concurrent.LinkedBlockingQueue;
43 import java.util.concurrent.ThreadPoolExecutor;
44 import java.util.concurrent.TimeUnit;
45 import java.util.concurrent.atomic.AtomicBoolean;
46 import java.util.concurrent.atomic.AtomicInteger;
47
48 import org.apache.commons.logging.Log;
49 import org.apache.commons.logging.LogFactory;
50 import org.apache.hadoop.classification.InterfaceAudience;
51 import org.apache.hadoop.classification.InterfaceStability;
52 import org.apache.hadoop.conf.Configuration;
53 import org.apache.hadoop.hbase.Chore;
54 import org.apache.hadoop.hbase.HBaseConfiguration;
55 import org.apache.hadoop.hbase.HConstants;
56 import org.apache.hadoop.hbase.HRegionInfo;
57 import org.apache.hadoop.hbase.HRegionLocation;
58 import org.apache.hadoop.hbase.HTableDescriptor;
59 import org.apache.hadoop.hbase.MasterNotRunningException;
60 import org.apache.hadoop.hbase.RegionTooBusyException;
61 import org.apache.hadoop.hbase.ServerName;
62 import org.apache.hadoop.hbase.Stoppable;
63 import org.apache.hadoop.hbase.TableName;
64 import org.apache.hadoop.hbase.TableNotEnabledException;
65 import org.apache.hadoop.hbase.TableNotFoundException;
66 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
67 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
68 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
69 import org.apache.hadoop.hbase.client.coprocessor.Batch;
70 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
71 import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
72 import org.apache.hadoop.hbase.ipc.RpcClient;
73 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
74 import org.apache.hadoop.hbase.protobuf.RequestConverter;
75 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
76 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
77 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
78 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
79 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnResponse;
80 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionResponse;
81 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceResponse;
82 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableResponse;
83 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnResponse;
84 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotResponse;
85 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableResponse;
86 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse;
87 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse;
88 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
89 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse;
90 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsResponse;
91 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
92 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
93 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest;
94 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse;
95 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
96 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse;
97 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
98 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
99 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceResponse;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnResponse;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableResponse;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionResponse;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionResponse;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterResponse;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionResponse;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.*;
116 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
117 import org.apache.hadoop.hbase.security.User;
118 import org.apache.hadoop.hbase.security.UserProvider;
119 import org.apache.hadoop.hbase.util.Bytes;
120 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
121 import org.apache.hadoop.hbase.util.ExceptionUtil;
122 import org.apache.hadoop.hbase.util.Threads;
123 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
124 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
125 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
126 import org.apache.hadoop.ipc.RemoteException;
127 import org.apache.zookeeper.KeeperException;
128
129 import com.google.common.annotations.VisibleForTesting;
130 import com.google.protobuf.BlockingRpcChannel;
131 import com.google.protobuf.RpcController;
132 import com.google.protobuf.ServiceException;
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196 @SuppressWarnings("serial")
197 @InterfaceAudience.Public
198 @InterfaceStability.Evolving
199 public class HConnectionManager {
200 static final Log LOG = LogFactory.getLog(HConnectionManager.class);
201
202 public static final String RETRIES_BY_SERVER_KEY = "hbase.client.retries.by.server";
203 private static final String CLIENT_NONCES_ENABLED_KEY = "hbase.client.nonces.enabled";
204
205
206
207
208 static final Map<HConnectionKey, HConnectionImplementation> CONNECTION_INSTANCES;
209
210 public static final int MAX_CACHED_CONNECTION_INSTANCES;
211
212
213
214
215
216 private static volatile NonceGenerator nonceGenerator = null;
217
218 private static Object nonceGeneratorCreateLock = new Object();
219
220 static {
221
222
223
224
225 MAX_CACHED_CONNECTION_INSTANCES = HBaseConfiguration.create().getInt(
226 HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1;
227 CONNECTION_INSTANCES = new LinkedHashMap<HConnectionKey, HConnectionImplementation>(
228 (int) (MAX_CACHED_CONNECTION_INSTANCES / 0.75F) + 1, 0.75F, true) {
229 @Override
230 protected boolean removeEldestEntry(
231 Map.Entry<HConnectionKey, HConnectionImplementation> eldest) {
232 return size() > MAX_CACHED_CONNECTION_INSTANCES;
233 }
234 };
235 }
236
237
238
239
240 private HConnectionManager() {
241 super();
242 }
243
244
245
246
247
248
249 @VisibleForTesting
250 public static NonceGenerator injectNonceGeneratorForTesting(
251 HConnection conn, NonceGenerator cnm) {
252 NonceGenerator ng = conn.getNonceGenerator();
253 LOG.warn("Nonce generator is being replaced by test code for " + cnm.getClass().getName());
254 ((HConnectionImplementation)conn).nonceGenerator = cnm;
255 return ng;
256 }
257
258
259
260
261
262
263
264
265
266
267 @Deprecated
268 public static HConnection getConnection(final Configuration conf)
269 throws IOException {
270 HConnectionKey connectionKey = new HConnectionKey(conf);
271 synchronized (CONNECTION_INSTANCES) {
272 HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);
273 if (connection == null) {
274 connection = (HConnectionImplementation)createConnection(conf, true);
275 CONNECTION_INSTANCES.put(connectionKey, connection);
276 } else if (connection.isClosed()) {
277 HConnectionManager.deleteConnection(connectionKey, true);
278 connection = (HConnectionImplementation)createConnection(conf, true);
279 CONNECTION_INSTANCES.put(connectionKey, connection);
280 }
281 connection.incCount();
282 return connection;
283 }
284 }
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306 public static HConnection createConnection(Configuration conf)
307 throws IOException {
308 UserProvider provider = UserProvider.instantiate(conf);
309 return createConnection(conf, false, null, provider.getCurrent());
310 }
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332 public static HConnection createConnection(Configuration conf, ExecutorService pool)
333 throws IOException {
334 UserProvider provider = UserProvider.instantiate(conf);
335 return createConnection(conf, false, pool, provider.getCurrent());
336 }
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358 public static HConnection createConnection(Configuration conf, User user)
359 throws IOException {
360 return createConnection(conf, false, null, user);
361 }
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384 public static HConnection createConnection(Configuration conf, ExecutorService pool, User user)
385 throws IOException {
386 return createConnection(conf, false, pool, user);
387 }
388
389 @Deprecated
390 static HConnection createConnection(final Configuration conf, final boolean managed)
391 throws IOException {
392 UserProvider provider = UserProvider.instantiate(conf);
393 return createConnection(conf, managed, null, provider.getCurrent());
394 }
395
396 @Deprecated
397 static HConnection createConnection(final Configuration conf, final boolean managed,
398 final ExecutorService pool, final User user)
399 throws IOException {
400 String className = conf.get("hbase.client.connection.impl",
401 HConnectionManager.HConnectionImplementation.class.getName());
402 Class<?> clazz = null;
403 try {
404 clazz = Class.forName(className);
405 } catch (ClassNotFoundException e) {
406 throw new IOException(e);
407 }
408 try {
409
410 Constructor<?> constructor =
411 clazz.getDeclaredConstructor(Configuration.class,
412 boolean.class, ExecutorService.class, User.class);
413 constructor.setAccessible(true);
414 return (HConnection) constructor.newInstance(conf, managed, pool, user);
415 } catch (Exception e) {
416 throw new IOException(e);
417 }
418 }
419
420
421
422
423
424
425
426
427
428 public static void deleteConnection(Configuration conf) {
429 deleteConnection(new HConnectionKey(conf), false);
430 }
431
432
433
434
435
436
437
438
439 public static void deleteStaleConnection(HConnection connection) {
440 deleteConnection(connection, true);
441 }
442
443
444
445
446
447
448
449 public static void deleteAllConnections(boolean staleConnection) {
450 synchronized (CONNECTION_INSTANCES) {
451 Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
452 connectionKeys.addAll(CONNECTION_INSTANCES.keySet());
453 for (HConnectionKey connectionKey : connectionKeys) {
454 deleteConnection(connectionKey, staleConnection);
455 }
456 CONNECTION_INSTANCES.clear();
457 }
458 }
459
460
461
462
463
464 @Deprecated
465 public static void deleteAllConnections() {
466 deleteAllConnections(false);
467 }
468
469
470 @Deprecated
471 private static void deleteConnection(HConnection connection, boolean staleConnection) {
472 synchronized (CONNECTION_INSTANCES) {
473 for (Entry<HConnectionKey, HConnectionImplementation> e: CONNECTION_INSTANCES.entrySet()) {
474 if (e.getValue() == connection) {
475 deleteConnection(e.getKey(), staleConnection);
476 break;
477 }
478 }
479 }
480 }
481
482 @Deprecated
483 private static void deleteConnection(HConnectionKey connectionKey, boolean staleConnection) {
484 synchronized (CONNECTION_INSTANCES) {
485 HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);
486 if (connection != null) {
487 connection.decCount();
488 if (connection.isZeroReference() || staleConnection) {
489 CONNECTION_INSTANCES.remove(connectionKey);
490 connection.internalClose();
491 }
492 } else {
493 LOG.error("Connection not found in the list, can't delete it "+
494 "(connection key=" + connectionKey + "). May be the key was modified?", new Exception());
495 }
496 }
497 }
498
499
500
501
502
503
504
505 static int getCachedRegionCount(Configuration conf, final TableName tableName)
506 throws IOException {
507 return execute(new HConnectable<Integer>(conf) {
508 @Override
509 public Integer connect(HConnection connection) {
510 return ((HConnectionImplementation)connection).getNumberOfCachedRegionLocations(tableName);
511 }
512 });
513 }
514
515
516
517
518
519
520
521 static boolean isRegionCached(Configuration conf,
522 final TableName tableName,
523 final byte[] row)
524 throws IOException {
525 return execute(new HConnectable<Boolean>(conf) {
526 @Override
527 public Boolean connect(HConnection connection) {
528 return ((HConnectionImplementation) connection).isRegionCached(tableName, row);
529 }
530 });
531 }
532
533
534
535
536
537
538
539
540
541
542
543 @InterfaceAudience.Private
544 public static <T> T execute(HConnectable<T> connectable) throws IOException {
545 if (connectable == null || connectable.conf == null) {
546 return null;
547 }
548 Configuration conf = connectable.conf;
549 HConnection connection = HConnectionManager.getConnection(conf);
550 boolean connectSucceeded = false;
551 try {
552 T returnValue = connectable.connect(connection);
553 connectSucceeded = true;
554 return returnValue;
555 } finally {
556 try {
557 connection.close();
558 } catch (Exception e) {
559 ExceptionUtil.rethrowIfInterrupt(e);
560 if (connectSucceeded) {
561 throw new IOException("The connection to " + connection
562 + " could not be deleted.", e);
563 }
564 }
565 }
566 }
567
568
569 @edu.umd.cs.findbugs.annotations.SuppressWarnings(
570 value="AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION",
571 justification="Access to the conncurrent hash map is under a lock so should be fine.")
572 static class HConnectionImplementation implements HConnection, Closeable {
573 static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);
574 private final long pause;
575 private final int numTries;
576 final int rpcTimeout;
577 private NonceGenerator nonceGenerator = null;
578 private final int prefetchRegionLimit;
579
580 private volatile boolean closed;
581 private volatile boolean aborted;
582
583
584 ClusterStatusListener clusterStatusListener;
585
586 private final Object userRegionLock = new Object();
587
588
589
590
591
592
593 private final Object masterAndZKLock = new Object();
594
595 private long keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
596 private final DelayedClosing delayedClosing =
597 DelayedClosing.createAndStart(this);
598
599
600
601 private volatile ExecutorService batchPool = null;
602 private volatile boolean cleanupPool = false;
603
604 private final Configuration conf;
605
606
607 private RpcClient rpcClient;
608
609
610
611
612 private final ConcurrentMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>
613 cachedRegionLocations =
614 new ConcurrentHashMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>();
615
616
617
618
619
620
621 private final Set<ServerName> cachedServers = new ConcurrentSkipListSet<ServerName>();
622
623
624
625 private final Set<Integer> regionCachePrefetchDisabledTables =
626 new CopyOnWriteArraySet<Integer>();
627
628 private int refCount;
629
630
631 private boolean managed;
632
633 private User user;
634
635
636
637
638 Registry registry;
639
640 HConnectionImplementation(Configuration conf, boolean managed) throws IOException {
641 this(conf, managed, null, null);
642 }
643
644
645
646
647
648
649
650
651
652
653
654
655 HConnectionImplementation(Configuration conf, boolean managed,
656 ExecutorService pool, User user) throws IOException {
657 this(conf);
658 this.user = user;
659 this.batchPool = pool;
660 this.managed = managed;
661 this.registry = setupRegistry();
662 retrieveClusterId();
663
664 this.rpcClient = new RpcClient(this.conf, this.clusterId);
665
666
667 boolean shouldListen = conf.getBoolean(HConstants.STATUS_PUBLISHED,
668 HConstants.STATUS_PUBLISHED_DEFAULT);
669 Class<? extends ClusterStatusListener.Listener> listenerClass =
670 conf.getClass(ClusterStatusListener.STATUS_LISTENER_CLASS,
671 ClusterStatusListener.DEFAULT_STATUS_LISTENER_CLASS,
672 ClusterStatusListener.Listener.class);
673 if (shouldListen) {
674 if (listenerClass == null) {
675 LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
676 ClusterStatusListener.STATUS_LISTENER_CLASS + " is not set - not listening status");
677 } else {
678 clusterStatusListener = new ClusterStatusListener(
679 new ClusterStatusListener.DeadServerHandler() {
680 @Override
681 public void newDead(ServerName sn) {
682 clearCaches(sn);
683 rpcClient.cancelConnections(sn.getHostname(), sn.getPort(),
684 new SocketException(sn.getServerName() +
685 " is dead: closing its connection."));
686 }
687 }, conf, listenerClass);
688 }
689 }
690 }
691
692
693 private static class NoNonceGenerator implements NonceGenerator {
694 @Override
695 public long getNonceGroup() {
696 return HConstants.NO_NONCE;
697 }
698 @Override
699 public long newNonce() {
700 return HConstants.NO_NONCE;
701 }
702 }
703
704
705
706
707 protected HConnectionImplementation(Configuration conf) {
708 this.conf = conf;
709 this.closed = false;
710 this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
711 HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
712 this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
713 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
714 this.rpcTimeout = conf.getInt(
715 HConstants.HBASE_RPC_TIMEOUT_KEY,
716 HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
717 if (conf.getBoolean(CLIENT_NONCES_ENABLED_KEY, true)) {
718 synchronized (HConnectionManager.nonceGeneratorCreateLock) {
719 if (HConnectionManager.nonceGenerator == null) {
720 HConnectionManager.nonceGenerator = new PerClientRandomNonceGenerator();
721 }
722 this.nonceGenerator = HConnectionManager.nonceGenerator;
723 }
724 } else {
725 this.nonceGenerator = new NoNonceGenerator();
726 }
727
728 this.prefetchRegionLimit = conf.getInt(
729 HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
730 HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
731 }
732
733 @Override
734 public HTableInterface getTable(String tableName) throws IOException {
735 return getTable(TableName.valueOf(tableName));
736 }
737
738 @Override
739 public HTableInterface getTable(byte[] tableName) throws IOException {
740 return getTable(TableName.valueOf(tableName));
741 }
742
743 @Override
744 public HTableInterface getTable(TableName tableName) throws IOException {
745 return getTable(tableName, getBatchPool());
746 }
747
748 @Override
749 public HTableInterface getTable(String tableName, ExecutorService pool) throws IOException {
750 return getTable(TableName.valueOf(tableName), pool);
751 }
752
753 @Override
754 public HTableInterface getTable(byte[] tableName, ExecutorService pool) throws IOException {
755 return getTable(TableName.valueOf(tableName), pool);
756 }
757
758 @Override
759 public HTableInterface getTable(TableName tableName, ExecutorService pool) throws IOException {
760 if (managed) {
761 throw new IOException("The connection has to be unmanaged.");
762 }
763 return new HTable(tableName, this, pool);
764 }
765
766 private ExecutorService getBatchPool() {
767 if (batchPool == null) {
768
769 synchronized (this) {
770 if (batchPool == null) {
771 int maxThreads = conf.getInt("hbase.hconnection.threads.max", 256);
772 int coreThreads = conf.getInt("hbase.hconnection.threads.core", 256);
773 if (maxThreads == 0) {
774 maxThreads = Runtime.getRuntime().availableProcessors() * 8;
775 }
776 if (coreThreads == 0) {
777 coreThreads = Runtime.getRuntime().availableProcessors() * 8;
778 }
779 long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60);
780 LinkedBlockingQueue<Runnable> workQueue =
781 new LinkedBlockingQueue<Runnable>(maxThreads *
782 conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
783 HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
784 ThreadPoolExecutor tpe = new ThreadPoolExecutor(
785 coreThreads,
786 maxThreads,
787 keepAliveTime,
788 TimeUnit.SECONDS,
789 workQueue,
790 Threads.newDaemonThreadFactory(toString() + "-shared-"));
791 tpe.allowCoreThreadTimeOut(true);
792 this.batchPool = tpe;
793 }
794 this.cleanupPool = true;
795 }
796 }
797 return this.batchPool;
798 }
799
800 protected ExecutorService getCurrentBatchPool() {
801 return batchPool;
802 }
803
804 private void shutdownBatchPool() {
805 if (this.cleanupPool && this.batchPool != null && !this.batchPool.isShutdown()) {
806 this.batchPool.shutdown();
807 try {
808 if (!this.batchPool.awaitTermination(10, TimeUnit.SECONDS)) {
809 this.batchPool.shutdownNow();
810 }
811 } catch (InterruptedException e) {
812 this.batchPool.shutdownNow();
813 }
814 }
815 }
816
817
818
819
820
821 private Registry setupRegistry() throws IOException {
822 String registryClass = this.conf.get("hbase.client.registry.impl",
823 ZooKeeperRegistry.class.getName());
824 Registry registry = null;
825 try {
826 registry = (Registry)Class.forName(registryClass).newInstance();
827 } catch (Throwable t) {
828 throw new IOException(t);
829 }
830 registry.init(this);
831 return registry;
832 }
833
834
835
836
837
838
839 RpcClient setRpcClient(final RpcClient rpcClient) {
840 RpcClient oldRpcClient = this.rpcClient;
841 this.rpcClient = rpcClient;
842 return oldRpcClient;
843 }
844
845
846
847
848
849 public String toString(){
850 return "hconnection-0x" + Integer.toHexString(hashCode());
851 }
852
853 protected String clusterId = null;
854
855 void retrieveClusterId() {
856 if (clusterId != null) return;
857 this.clusterId = this.registry.getClusterId();
858 if (clusterId == null) {
859 clusterId = HConstants.CLUSTER_ID_DEFAULT;
860 LOG.debug("clusterid came back null, using default " + clusterId);
861 }
862 }
863
864 @Override
865 public Configuration getConfiguration() {
866 return this.conf;
867 }
868
869 private void checkIfBaseNodeAvailable(ZooKeeperWatcher zkw)
870 throws MasterNotRunningException {
871 String errorMsg;
872 try {
873 if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {
874 errorMsg = "The node " + zkw.baseZNode+" is not in ZooKeeper. "
875 + "It should have been written by the master. "
876 + "Check the value configured in 'zookeeper.znode.parent'. "
877 + "There could be a mismatch with the one configured in the master.";
878 LOG.error(errorMsg);
879 throw new MasterNotRunningException(errorMsg);
880 }
881 } catch (KeeperException e) {
882 errorMsg = "Can't get connection to ZooKeeper: " + e.getMessage();
883 LOG.error(errorMsg);
884 throw new MasterNotRunningException(errorMsg, e);
885 }
886 }
887
888
889
890
891
892
893 @Override
894 public boolean isMasterRunning()
895 throws MasterNotRunningException, ZooKeeperConnectionException {
896
897
898
899 MasterKeepAliveConnection m = getKeepAliveMasterService();
900 m.close();
901 return true;
902 }
903
904 @Override
905 public HRegionLocation getRegionLocation(final TableName tableName,
906 final byte [] row, boolean reload)
907 throws IOException {
908 return reload? relocateRegion(tableName, row): locateRegion(tableName, row);
909 }
910
911 @Override
912 public HRegionLocation getRegionLocation(final byte[] tableName,
913 final byte [] row, boolean reload)
914 throws IOException {
915 return getRegionLocation(TableName.valueOf(tableName), row, reload);
916 }
917
918 @Override
919 public boolean isTableEnabled(TableName tableName) throws IOException {
920 return this.registry.isTableOnlineState(tableName, true);
921 }
922
923 @Override
924 public boolean isTableEnabled(byte[] tableName) throws IOException {
925 return isTableEnabled(TableName.valueOf(tableName));
926 }
927
928 @Override
929 public boolean isTableDisabled(TableName tableName) throws IOException {
930 return this.registry.isTableOnlineState(tableName, false);
931 }
932
933 @Override
934 public boolean isTableDisabled(byte[] tableName) throws IOException {
935 return isTableDisabled(TableName.valueOf(tableName));
936 }
937
938 @Override
939 public boolean isTableAvailable(final TableName tableName) throws IOException {
940 final AtomicBoolean available = new AtomicBoolean(true);
941 final AtomicInteger regionCount = new AtomicInteger(0);
942 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
943 @Override
944 public boolean processRow(Result row) throws IOException {
945 HRegionInfo info = MetaScanner.getHRegionInfo(row);
946 if (info != null && !info.isSplitParent()) {
947 if (tableName.equals(info.getTable())) {
948 ServerName server = HRegionInfo.getServerName(row);
949 if (server == null) {
950 available.set(false);
951 return false;
952 }
953 regionCount.incrementAndGet();
954 } else if (tableName.compareTo(info.getTable()) < 0) {
955
956 return false;
957 }
958 }
959 return true;
960 }
961 };
962 MetaScanner.metaScan(conf, this, visitor, tableName);
963 return available.get() && (regionCount.get() > 0);
964 }
965
966 @Override
967 public boolean isTableAvailable(final byte[] tableName) throws IOException {
968 return isTableAvailable(TableName.valueOf(tableName));
969 }
970
971 @Override
972 public boolean isTableAvailable(final TableName tableName, final byte[][] splitKeys)
973 throws IOException {
974 final AtomicBoolean available = new AtomicBoolean(true);
975 final AtomicInteger regionCount = new AtomicInteger(0);
976 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
977 @Override
978 public boolean processRow(Result row) throws IOException {
979 HRegionInfo info = MetaScanner.getHRegionInfo(row);
980 if (info != null && !info.isSplitParent()) {
981 if (tableName.equals(info.getTable())) {
982 ServerName server = HRegionInfo.getServerName(row);
983 if (server == null) {
984 available.set(false);
985 return false;
986 }
987 if (!Bytes.equals(info.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
988 for (byte[] splitKey : splitKeys) {
989
990 if (Bytes.equals(info.getStartKey(), splitKey)) {
991 regionCount.incrementAndGet();
992 break;
993 }
994 }
995 } else {
996
997 regionCount.incrementAndGet();
998 }
999 } else if (tableName.compareTo(info.getTable()) < 0) {
1000
1001 return false;
1002 }
1003 }
1004 return true;
1005 }
1006 };
1007 MetaScanner.metaScan(conf, this, visitor, tableName);
1008
1009 return available.get() && (regionCount.get() == splitKeys.length + 1);
1010 }
1011
1012 @Override
1013 public boolean isTableAvailable(final byte[] tableName, final byte[][] splitKeys)
1014 throws IOException {
1015 return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1016 }
1017
1018 @Override
1019 public HRegionLocation locateRegion(final byte[] regionName) throws IOException {
1020 return locateRegion(HRegionInfo.getTable(regionName),
1021 HRegionInfo.getStartKey(regionName), false, true);
1022 }
1023
1024 @Override
1025 public boolean isDeadServer(ServerName sn) {
1026 if (clusterStatusListener == null) {
1027 return false;
1028 } else {
1029 return clusterStatusListener.isDeadServer(sn);
1030 }
1031 }
1032
1033 @Override
1034 public List<HRegionLocation> locateRegions(final TableName tableName)
1035 throws IOException {
1036 return locateRegions (tableName, false, true);
1037 }
1038
1039 @Override
1040 public List<HRegionLocation> locateRegions(final byte[] tableName)
1041 throws IOException {
1042 return locateRegions(TableName.valueOf(tableName));
1043 }
1044
1045 @Override
1046 public List<HRegionLocation> locateRegions(final TableName tableName,
1047 final boolean useCache, final boolean offlined) throws IOException {
1048 NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, this,
1049 tableName, offlined);
1050 final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();
1051 for (HRegionInfo regionInfo : regions.keySet()) {
1052 locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true));
1053 }
1054 return locations;
1055 }
1056
1057 @Override
1058 public List<HRegionLocation> locateRegions(final byte[] tableName,
1059 final boolean useCache, final boolean offlined) throws IOException {
1060 return locateRegions(TableName.valueOf(tableName), useCache, offlined);
1061 }
1062
1063 @Override
1064 public HRegionLocation locateRegion(final TableName tableName,
1065 final byte [] row)
1066 throws IOException{
1067 return locateRegion(tableName, row, true, true);
1068 }
1069
1070 @Override
1071 public HRegionLocation locateRegion(final byte[] tableName,
1072 final byte [] row)
1073 throws IOException{
1074 return locateRegion(TableName.valueOf(tableName), row);
1075 }
1076
1077 @Override
1078 public HRegionLocation relocateRegion(final TableName tableName,
1079 final byte [] row) throws IOException{
1080
1081
1082
1083 if (isTableDisabled(tableName)) {
1084 throw new TableNotEnabledException(tableName.getNameAsString() + " is disabled.");
1085 }
1086
1087 return locateRegion(tableName, row, false, true);
1088 }
1089
1090 @Override
1091 public HRegionLocation relocateRegion(final byte[] tableName,
1092 final byte [] row) throws IOException {
1093 return relocateRegion(TableName.valueOf(tableName), row);
1094 }
1095
1096
1097 private HRegionLocation locateRegion(final TableName tableName,
1098 final byte [] row, boolean useCache, boolean retry)
1099 throws IOException {
1100 if (this.closed) throw new IOException(toString() + " closed");
1101 if (tableName== null || tableName.getName().length == 0) {
1102 throw new IllegalArgumentException(
1103 "table name cannot be null or zero length");
1104 }
1105
1106 if (tableName.equals(TableName.META_TABLE_NAME)) {
1107 return this.registry.getMetaRegionLocation();
1108 } else {
1109
1110 return locateRegionInMeta(TableName.META_TABLE_NAME, tableName, row,
1111 useCache, userRegionLock, retry);
1112 }
1113 }
1114
1115
1116
1117
1118
1119
1120 private void prefetchRegionCache(final TableName tableName,
1121 final byte[] row) {
1122
1123
1124 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
1125 public boolean processRow(Result result) throws IOException {
1126 try {
1127 HRegionInfo regionInfo = MetaScanner.getHRegionInfo(result);
1128 if (regionInfo == null) {
1129 return true;
1130 }
1131
1132
1133 if (!regionInfo.getTable().equals(tableName)) {
1134 return false;
1135 }
1136 if (regionInfo.isOffline()) {
1137
1138 return true;
1139 }
1140
1141 ServerName serverName = HRegionInfo.getServerName(result);
1142 if (serverName == null) {
1143 return true;
1144 }
1145
1146 long seqNum = HRegionInfo.getSeqNumDuringOpen(result);
1147 HRegionLocation loc = new HRegionLocation(regionInfo, serverName, seqNum);
1148
1149 cacheLocation(tableName, null, loc);
1150 return true;
1151 } catch (RuntimeException e) {
1152 throw new IOException(e);
1153 }
1154 }
1155 };
1156 try {
1157
1158 MetaScanner.metaScan(conf, this, visitor, tableName, row,
1159 this.prefetchRegionLimit, TableName.META_TABLE_NAME);
1160 } catch (IOException e) {
1161 if (ExceptionUtil.isInterrupt(e)) {
1162 Thread.currentThread().interrupt();
1163 } else {
1164 LOG.warn("Encountered problems when prefetch hbase:meta table: ", e);
1165 }
1166 }
1167 }
1168
1169
1170
1171
1172
1173 private HRegionLocation locateRegionInMeta(final TableName parentTable,
1174 final TableName tableName, final byte [] row, boolean useCache,
1175 Object regionLockObject, boolean retry)
1176 throws IOException {
1177 HRegionLocation location;
1178
1179
1180 if (useCache) {
1181 location = getCachedLocation(tableName, row);
1182 if (location != null) {
1183 return location;
1184 }
1185 }
1186 int localNumRetries = retry ? numTries : 1;
1187
1188
1189
1190 byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
1191 HConstants.NINES, false);
1192 for (int tries = 0; true; tries++) {
1193 if (tries >= localNumRetries) {
1194 throw new NoServerForRegionException("Unable to find region for "
1195 + Bytes.toStringBinary(row) + " after " + numTries + " tries.");
1196 }
1197
1198 HRegionLocation metaLocation = null;
1199 try {
1200
1201 metaLocation = locateRegion(parentTable, metaKey, true, false);
1202
1203 if (metaLocation == null) continue;
1204 ClientService.BlockingInterface service = getClient(metaLocation.getServerName());
1205
1206 Result regionInfoRow;
1207
1208
1209
1210 if (useCache) {
1211 if (TableName.META_TABLE_NAME.equals(parentTable) &&
1212 getRegionCachePrefetch(tableName)) {
1213 synchronized (regionLockObject) {
1214
1215
1216 location = getCachedLocation(tableName, row);
1217 if (location != null) {
1218 return location;
1219 }
1220
1221
1222 prefetchRegionCache(tableName, row);
1223 }
1224 }
1225 location = getCachedLocation(tableName, row);
1226 if (location != null) {
1227 return location;
1228 }
1229 } else {
1230
1231
1232 forceDeleteCachedLocation(tableName, row);
1233 }
1234
1235
1236 regionInfoRow = ProtobufUtil.getRowOrBefore(service,
1237 metaLocation.getRegionInfo().getRegionName(), metaKey,
1238 HConstants.CATALOG_FAMILY);
1239
1240 if (regionInfoRow == null) {
1241 throw new TableNotFoundException(tableName);
1242 }
1243
1244
1245 HRegionInfo regionInfo = MetaScanner.getHRegionInfo(regionInfoRow);
1246 if (regionInfo == null) {
1247 throw new IOException("HRegionInfo was null or empty in " +
1248 parentTable + ", row=" + regionInfoRow);
1249 }
1250
1251
1252 if (!regionInfo.getTable().equals(tableName)) {
1253 throw new TableNotFoundException(
1254 "Table '" + tableName + "' was not found, got: " +
1255 regionInfo.getTable() + ".");
1256 }
1257 if (regionInfo.isSplit()) {
1258 throw new RegionOfflineException("the only available region for" +
1259 " the required row is a split parent," +
1260 " the daughters should be online soon: " +
1261 regionInfo.getRegionNameAsString());
1262 }
1263 if (regionInfo.isOffline()) {
1264 throw new RegionOfflineException("the region is offline, could" +
1265 " be caused by a disable table call: " +
1266 regionInfo.getRegionNameAsString());
1267 }
1268
1269 ServerName serverName = HRegionInfo.getServerName(regionInfoRow);
1270 if (serverName == null) {
1271 throw new NoServerForRegionException("No server address listed " +
1272 "in " + parentTable + " for region " +
1273 regionInfo.getRegionNameAsString() + " containing row " +
1274 Bytes.toStringBinary(row));
1275 }
1276
1277 if (isDeadServer(serverName)){
1278 throw new RegionServerStoppedException("hbase:meta says the region "+
1279 regionInfo.getRegionNameAsString()+" is managed by the server " + serverName +
1280 ", but it is dead.");
1281 }
1282
1283
1284 location = new HRegionLocation(regionInfo, serverName,
1285 HRegionInfo.getSeqNumDuringOpen(regionInfoRow));
1286 cacheLocation(tableName, null, location);
1287 return location;
1288 } catch (TableNotFoundException e) {
1289
1290
1291
1292 throw e;
1293 } catch (IOException e) {
1294 ExceptionUtil.rethrowIfInterrupt(e);
1295
1296 if (e instanceof RemoteException) {
1297 e = ((RemoteException)e).unwrapRemoteException();
1298 }
1299 if (tries < numTries - 1) {
1300 if (LOG.isDebugEnabled()) {
1301 LOG.debug("locateRegionInMeta parentTable=" +
1302 parentTable + ", metaLocation=" +
1303 ((metaLocation == null)? "null": "{" + metaLocation + "}") +
1304 ", attempt=" + tries + " of " +
1305 this.numTries + " failed; retrying after sleep of " +
1306 ConnectionUtils.getPauseTime(this.pause, tries) + " because: " + e.getMessage());
1307 }
1308 } else {
1309 throw e;
1310 }
1311
1312 if(!(e instanceof RegionOfflineException ||
1313 e instanceof NoServerForRegionException)) {
1314 relocateRegion(parentTable, metaKey);
1315 }
1316 }
1317 try{
1318 Thread.sleep(ConnectionUtils.getPauseTime(this.pause, tries));
1319 } catch (InterruptedException e) {
1320 throw new InterruptedIOException("Giving up trying to location region in " +
1321 "meta: thread is interrupted.");
1322 }
1323 }
1324 }
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334 HRegionLocation getCachedLocation(final TableName tableName,
1335 final byte [] row) {
1336 ConcurrentSkipListMap<byte[], HRegionLocation> tableLocations =
1337 getTableLocations(tableName);
1338
1339 Entry<byte[], HRegionLocation> e = tableLocations.floorEntry(row);
1340 if (e == null) {
1341 return null;
1342 }
1343 HRegionLocation possibleRegion = e.getValue();
1344
1345
1346
1347
1348
1349
1350 byte[] endKey = possibleRegion.getRegionInfo().getEndKey();
1351 if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
1352 tableName.getRowComparator().compareRows(
1353 endKey, 0, endKey.length, row, 0, row.length) > 0) {
1354 return possibleRegion;
1355 }
1356
1357
1358 return null;
1359 }
1360
1361
1362
1363
1364
1365
1366 void forceDeleteCachedLocation(final TableName tableName, final byte [] row) {
1367 HRegionLocation rl = null;
1368 Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1369
1370
1371 rl = getCachedLocation(tableName, row);
1372 if (rl != null) {
1373 tableLocations.remove(rl.getRegionInfo().getStartKey());
1374 }
1375 if ((rl != null) && LOG.isDebugEnabled()) {
1376 LOG.debug("Removed " + rl.getHostname() + ":" + rl.getPort()
1377 + " as a location of " + rl.getRegionInfo().getRegionNameAsString() +
1378 " for tableName=" + tableName + " from cache");
1379 }
1380 }
1381
1382
1383
1384
1385 @Override
1386 public void clearCaches(final ServerName serverName) {
1387 if (!this.cachedServers.contains(serverName)) {
1388 return;
1389 }
1390
1391 boolean deletedSomething = false;
1392 synchronized (this.cachedServers) {
1393
1394
1395
1396
1397 if (!this.cachedServers.contains(serverName)) {
1398 return;
1399 }
1400 for (Map<byte[], HRegionLocation> tableLocations : cachedRegionLocations.values()) {
1401 for (Entry<byte[], HRegionLocation> e : tableLocations.entrySet()) {
1402 HRegionLocation value = e.getValue();
1403 if (value != null
1404 && serverName.equals(value.getServerName())) {
1405 tableLocations.remove(e.getKey());
1406 deletedSomething = true;
1407 }
1408 }
1409 }
1410 this.cachedServers.remove(serverName);
1411 }
1412 if (deletedSomething && LOG.isDebugEnabled()) {
1413 LOG.debug("Removed all cached region locations that map to " + serverName);
1414 }
1415 }
1416
1417
1418
1419
1420
1421 private ConcurrentSkipListMap<byte[], HRegionLocation> getTableLocations(
1422 final TableName tableName) {
1423
1424 ConcurrentSkipListMap<byte[], HRegionLocation> result;
1425 result = this.cachedRegionLocations.get(tableName);
1426
1427 if (result == null) {
1428 result = new ConcurrentSkipListMap<byte[], HRegionLocation>(Bytes.BYTES_COMPARATOR);
1429 ConcurrentSkipListMap<byte[], HRegionLocation> old =
1430 this.cachedRegionLocations.putIfAbsent(tableName, result);
1431 if (old != null) {
1432 return old;
1433 }
1434 }
1435 return result;
1436 }
1437
1438 @Override
1439 public void clearRegionCache() {
1440 this.cachedRegionLocations.clear();
1441 this.cachedServers.clear();
1442 }
1443
1444 @Override
1445 public void clearRegionCache(final TableName tableName) {
1446 this.cachedRegionLocations.remove(tableName);
1447 }
1448
1449 @Override
1450 public void clearRegionCache(final byte[] tableName) {
1451 clearRegionCache(TableName.valueOf(tableName));
1452 }
1453
1454
1455
1456
1457
1458
1459
1460 private void cacheLocation(final TableName tableName, final HRegionLocation source,
1461 final HRegionLocation location) {
1462 boolean isFromMeta = (source == null);
1463 byte [] startKey = location.getRegionInfo().getStartKey();
1464 ConcurrentMap<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1465 HRegionLocation oldLocation = tableLocations.putIfAbsent(startKey, location);
1466 boolean isNewCacheEntry = (oldLocation == null);
1467 if (isNewCacheEntry) {
1468 cachedServers.add(location.getServerName());
1469 return;
1470 }
1471 boolean updateCache;
1472
1473 if (oldLocation.equals(source)) {
1474 updateCache = true;
1475 } else {
1476 long newLocationSeqNum = location.getSeqNum();
1477
1478
1479 boolean isStaleMetaRecord = isFromMeta && (oldLocation.getSeqNum() > newLocationSeqNum);
1480
1481
1482
1483
1484
1485 boolean isStaleRedirect = !isFromMeta && (oldLocation.getSeqNum() >= newLocationSeqNum);
1486 boolean isStaleUpdate = (isStaleMetaRecord || isStaleRedirect);
1487 updateCache = (!isStaleUpdate);
1488 }
1489 if (updateCache) {
1490 tableLocations.replace(startKey, oldLocation, location);
1491 cachedServers.add(location.getServerName());
1492 }
1493 }
1494
1495
1496 private final ConcurrentHashMap<String, Object> stubs =
1497 new ConcurrentHashMap<String, Object>();
1498
1499 private final ConcurrentHashMap<String, String> connectionLock =
1500 new ConcurrentHashMap<String, String>();
1501
1502
1503
1504
1505 static class MasterServiceState {
1506 HConnection connection;
1507 MasterService.BlockingInterface stub;
1508 int userCount;
1509 long keepAliveUntil = Long.MAX_VALUE;
1510
1511 MasterServiceState (final HConnection connection) {
1512 super();
1513 this.connection = connection;
1514 }
1515
1516 @Override
1517 public String toString() {
1518 return "MasterService";
1519 }
1520
1521 Object getStub() {
1522 return this.stub;
1523 }
1524
1525 void clearStub() {
1526 this.stub = null;
1527 }
1528
1529 boolean isMasterRunning() throws ServiceException {
1530 IsMasterRunningResponse response =
1531 this.stub.isMasterRunning(null, RequestConverter.buildIsMasterRunningRequest());
1532 return response != null? response.getIsMasterRunning(): false;
1533 }
1534 }
1535
1536
1537
1538
1539
1540
1541 abstract class StubMaker {
1542
1543
1544
1545 protected abstract String getServiceName();
1546
1547
1548
1549
1550
1551 protected abstract Object makeStub(final BlockingRpcChannel channel);
1552
1553
1554
1555
1556
1557 protected abstract void isMasterRunning() throws ServiceException;
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567 private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
1568 ZooKeeperKeepAliveConnection zkw;
1569 try {
1570 zkw = getKeepAliveZooKeeperWatcher();
1571 } catch (IOException e) {
1572 ExceptionUtil.rethrowIfInterrupt(e);
1573 throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
1574 }
1575 try {
1576 checkIfBaseNodeAvailable(zkw);
1577 ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
1578 if (sn == null) {
1579 String msg = "ZooKeeper available but no active master location found";
1580 LOG.info(msg);
1581 throw new MasterNotRunningException(msg);
1582 }
1583 if (isDeadServer(sn)) {
1584 throw new MasterNotRunningException(sn + " is dead.");
1585 }
1586
1587 String key = getStubKey(getServiceName(), sn.getHostAndPort());
1588 connectionLock.putIfAbsent(key, key);
1589 Object stub = null;
1590 synchronized (connectionLock.get(key)) {
1591 stub = stubs.get(key);
1592 if (stub == null) {
1593 BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn,
1594 user, rpcTimeout);
1595 stub = makeStub(channel);
1596 isMasterRunning();
1597 stubs.put(key, stub);
1598 }
1599 }
1600 return stub;
1601 } finally {
1602 zkw.close();
1603 }
1604 }
1605
1606
1607
1608
1609
1610
1611 @edu.umd.cs.findbugs.annotations.SuppressWarnings (value="SWL_SLEEP_WITH_LOCK_HELD")
1612 Object makeStub() throws MasterNotRunningException {
1613
1614
1615 synchronized (masterAndZKLock) {
1616 Exception exceptionCaught = null;
1617 Object stub = null;
1618 int tries = 0;
1619 while (!closed && stub == null) {
1620 tries++;
1621 try {
1622 stub = makeStubNoRetries();
1623 } catch (IOException e) {
1624 exceptionCaught = e;
1625 } catch (KeeperException e) {
1626 exceptionCaught = e;
1627 } catch (ServiceException e) {
1628 exceptionCaught = e;
1629 }
1630
1631 if (exceptionCaught != null)
1632
1633 if (tries < numTries && !ExceptionUtil.isInterrupt(exceptionCaught)) {
1634
1635 long pauseTime = ConnectionUtils.getPauseTime(pause, tries - 1);
1636 LOG.info("getMaster attempt " + tries + " of " + numTries +
1637 " failed; retrying after sleep of " + pauseTime + ", exception=" +
1638 exceptionCaught);
1639
1640 try {
1641 Thread.sleep(pauseTime);
1642 } catch (InterruptedException e) {
1643 throw new MasterNotRunningException(
1644 "Thread was interrupted while trying to connect to master.", e);
1645 }
1646 } else {
1647
1648 LOG.info("getMaster attempt " + tries + " of " + numTries +
1649 " failed; no more retrying.", exceptionCaught);
1650 throw new MasterNotRunningException(exceptionCaught);
1651 }
1652 }
1653
1654 if (stub == null) {
1655
1656 throw new MasterNotRunningException("Connection was closed while trying to get master");
1657 }
1658 return stub;
1659 }
1660 }
1661 }
1662
1663
1664
1665
1666 class MasterServiceStubMaker extends StubMaker {
1667 private MasterService.BlockingInterface stub;
1668 @Override
1669 protected String getServiceName() {
1670 return MasterService.getDescriptor().getName();
1671 }
1672
1673 @Override
1674 @edu.umd.cs.findbugs.annotations.SuppressWarnings("SWL_SLEEP_WITH_LOCK_HELD")
1675 MasterService.BlockingInterface makeStub() throws MasterNotRunningException {
1676 return (MasterService.BlockingInterface)super.makeStub();
1677 }
1678
1679 @Override
1680 protected Object makeStub(BlockingRpcChannel channel) {
1681 this.stub = MasterService.newBlockingStub(channel);
1682 return this.stub;
1683 }
1684
1685 @Override
1686 protected void isMasterRunning() throws ServiceException {
1687 this.stub.isMasterRunning(null, RequestConverter.buildIsMasterRunningRequest());
1688 }
1689 }
1690
1691 @Override
1692 public AdminService.BlockingInterface getAdmin(final ServerName serverName)
1693 throws IOException {
1694 return getAdmin(serverName, false);
1695 }
1696
1697 @Override
1698
1699 public AdminService.BlockingInterface getAdmin(final ServerName serverName,
1700 final boolean master)
1701 throws IOException {
1702 if (isDeadServer(serverName)) {
1703 throw new RegionServerStoppedException(serverName + " is dead.");
1704 }
1705 String key = getStubKey(AdminService.BlockingInterface.class.getName(),
1706 serverName.getHostAndPort());
1707 this.connectionLock.putIfAbsent(key, key);
1708 AdminService.BlockingInterface stub = null;
1709 synchronized (this.connectionLock.get(key)) {
1710 stub = (AdminService.BlockingInterface)this.stubs.get(key);
1711 if (stub == null) {
1712 BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName,
1713 user, this.rpcTimeout);
1714 stub = AdminService.newBlockingStub(channel);
1715 this.stubs.put(key, stub);
1716 }
1717 }
1718 return stub;
1719 }
1720
1721 @Override
1722 public ClientService.BlockingInterface getClient(final ServerName sn)
1723 throws IOException {
1724 if (isDeadServer(sn)) {
1725 throw new RegionServerStoppedException(sn + " is dead.");
1726 }
1727 String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort());
1728 this.connectionLock.putIfAbsent(key, key);
1729 ClientService.BlockingInterface stub = null;
1730 synchronized (this.connectionLock.get(key)) {
1731 stub = (ClientService.BlockingInterface)this.stubs.get(key);
1732 if (stub == null) {
1733 BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(sn,
1734 user, this.rpcTimeout);
1735 stub = ClientService.newBlockingStub(channel);
1736
1737
1738 this.stubs.put(key, stub);
1739 }
1740 }
1741 return stub;
1742 }
1743
1744 static String getStubKey(final String serviceName, final String rsHostnamePort) {
1745 return serviceName + "@" + rsHostnamePort;
1746 }
1747
1748 private ZooKeeperKeepAliveConnection keepAliveZookeeper;
1749 private int keepAliveZookeeperUserCount;
1750 private boolean canCloseZKW = true;
1751
1752
1753 private static final long keepAlive = 5 * 60 * 1000;
1754
1755
1756
1757
1758
1759 ZooKeeperKeepAliveConnection getKeepAliveZooKeeperWatcher()
1760 throws IOException {
1761 synchronized (masterAndZKLock) {
1762 if (keepAliveZookeeper == null) {
1763 if (this.closed) {
1764 throw new IOException(toString() + " closed");
1765 }
1766
1767
1768 keepAliveZookeeper = new ZooKeeperKeepAliveConnection(conf, this.toString(), this);
1769 }
1770 keepAliveZookeeperUserCount++;
1771 keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1772 return keepAliveZookeeper;
1773 }
1774 }
1775
1776 void releaseZooKeeperWatcher(final ZooKeeperWatcher zkw) {
1777 if (zkw == null){
1778 return;
1779 }
1780 synchronized (masterAndZKLock) {
1781 --keepAliveZookeeperUserCount;
1782 if (keepAliveZookeeperUserCount <= 0 ){
1783 keepZooKeeperWatcherAliveUntil = System.currentTimeMillis() + keepAlive;
1784 }
1785 }
1786 }
1787
1788
1789
1790
1791
1792
1793
1794
1795 private static class DelayedClosing extends Chore implements Stoppable {
1796 private HConnectionImplementation hci;
1797 Stoppable stoppable;
1798
1799 private DelayedClosing(
1800 HConnectionImplementation hci, Stoppable stoppable){
1801 super(
1802 "ZooKeeperWatcher and Master delayed closing for connection "+hci,
1803 60*1000,
1804 stoppable);
1805 this.hci = hci;
1806 this.stoppable = stoppable;
1807 }
1808
1809 static DelayedClosing createAndStart(HConnectionImplementation hci){
1810 Stoppable stoppable = new Stoppable() {
1811 private volatile boolean isStopped = false;
1812 @Override public void stop(String why) { isStopped = true;}
1813 @Override public boolean isStopped() {return isStopped;}
1814 };
1815
1816 return new DelayedClosing(hci, stoppable);
1817 }
1818
1819 protected void closeMasterProtocol(MasterServiceState protocolState) {
1820 if (System.currentTimeMillis() > protocolState.keepAliveUntil) {
1821 hci.closeMasterService(protocolState);
1822 protocolState.keepAliveUntil = Long.MAX_VALUE;
1823 }
1824 }
1825
1826 @Override
1827 protected void chore() {
1828 synchronized (hci.masterAndZKLock) {
1829 if (hci.canCloseZKW) {
1830 if (System.currentTimeMillis() >
1831 hci.keepZooKeeperWatcherAliveUntil) {
1832
1833 hci.closeZooKeeperWatcher();
1834 hci.keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1835 }
1836 }
1837 closeMasterProtocol(hci.masterServiceState);
1838 closeMasterProtocol(hci.masterServiceState);
1839 }
1840 }
1841
1842 @Override
1843 public void stop(String why) {
1844 stoppable.stop(why);
1845 }
1846
1847 @Override
1848 public boolean isStopped() {
1849 return stoppable.isStopped();
1850 }
1851 }
1852
1853 private void closeZooKeeperWatcher() {
1854 synchronized (masterAndZKLock) {
1855 if (keepAliveZookeeper != null) {
1856 LOG.info("Closing zookeeper sessionid=0x" +
1857 Long.toHexString(
1858 keepAliveZookeeper.getRecoverableZooKeeper().getSessionId()));
1859 keepAliveZookeeper.internalClose();
1860 keepAliveZookeeper = null;
1861 }
1862 keepAliveZookeeperUserCount = 0;
1863 }
1864 }
1865
1866 final MasterServiceState masterServiceState = new MasterServiceState(this);
1867
1868 @Override
1869 public MasterService.BlockingInterface getMaster() throws MasterNotRunningException {
1870 return getKeepAliveMasterService();
1871 }
1872
1873 private void resetMasterServiceState(final MasterServiceState mss) {
1874 mss.userCount++;
1875 mss.keepAliveUntil = Long.MAX_VALUE;
1876 }
1877
1878 @Override
1879 public MasterKeepAliveConnection getKeepAliveMasterService()
1880 throws MasterNotRunningException {
1881 synchronized (masterAndZKLock) {
1882 if (!isKeepAliveMasterConnectedAndRunning(this.masterServiceState)) {
1883 MasterServiceStubMaker stubMaker = new MasterServiceStubMaker();
1884 this.masterServiceState.stub = stubMaker.makeStub();
1885 }
1886 resetMasterServiceState(this.masterServiceState);
1887 }
1888
1889 final MasterService.BlockingInterface stub = this.masterServiceState.stub;
1890 return new MasterKeepAliveConnection() {
1891 MasterServiceState mss = masterServiceState;
1892 @Override
1893 public AddColumnResponse addColumn(RpcController controller, AddColumnRequest request)
1894 throws ServiceException {
1895 return stub.addColumn(controller, request);
1896 }
1897
1898 @Override
1899 public DeleteColumnResponse deleteColumn(RpcController controller,
1900 DeleteColumnRequest request)
1901 throws ServiceException {
1902 return stub.deleteColumn(controller, request);
1903 }
1904
1905 @Override
1906 public ModifyColumnResponse modifyColumn(RpcController controller,
1907 ModifyColumnRequest request)
1908 throws ServiceException {
1909 return stub.modifyColumn(controller, request);
1910 }
1911
1912 @Override
1913 public MoveRegionResponse moveRegion(RpcController controller,
1914 MoveRegionRequest request) throws ServiceException {
1915 return stub.moveRegion(controller, request);
1916 }
1917
1918 @Override
1919 public DispatchMergingRegionsResponse dispatchMergingRegions(
1920 RpcController controller, DispatchMergingRegionsRequest request)
1921 throws ServiceException {
1922 return stub.dispatchMergingRegions(controller, request);
1923 }
1924
1925 @Override
1926 public AssignRegionResponse assignRegion(RpcController controller,
1927 AssignRegionRequest request) throws ServiceException {
1928 return stub.assignRegion(controller, request);
1929 }
1930
1931 @Override
1932 public UnassignRegionResponse unassignRegion(RpcController controller,
1933 UnassignRegionRequest request) throws ServiceException {
1934 return stub.unassignRegion(controller, request);
1935 }
1936
1937 @Override
1938 public OfflineRegionResponse offlineRegion(RpcController controller,
1939 OfflineRegionRequest request) throws ServiceException {
1940 return stub.offlineRegion(controller, request);
1941 }
1942
1943 @Override
1944 public DeleteTableResponse deleteTable(RpcController controller,
1945 DeleteTableRequest request) throws ServiceException {
1946 return stub.deleteTable(controller, request);
1947 }
1948
1949 @Override
1950 public EnableTableResponse enableTable(RpcController controller,
1951 EnableTableRequest request) throws ServiceException {
1952 return stub.enableTable(controller, request);
1953 }
1954
1955 @Override
1956 public DisableTableResponse disableTable(RpcController controller,
1957 DisableTableRequest request) throws ServiceException {
1958 return stub.disableTable(controller, request);
1959 }
1960
1961 @Override
1962 public ModifyTableResponse modifyTable(RpcController controller,
1963 ModifyTableRequest request) throws ServiceException {
1964 return stub.modifyTable(controller, request);
1965 }
1966
1967 @Override
1968 public CreateTableResponse createTable(RpcController controller,
1969 CreateTableRequest request) throws ServiceException {
1970 return stub.createTable(controller, request);
1971 }
1972
1973 @Override
1974 public ShutdownResponse shutdown(RpcController controller,
1975 ShutdownRequest request) throws ServiceException {
1976 return stub.shutdown(controller, request);
1977 }
1978
1979 @Override
1980 public StopMasterResponse stopMaster(RpcController controller,
1981 StopMasterRequest request) throws ServiceException {
1982 return stub.stopMaster(controller, request);
1983 }
1984
1985 @Override
1986 public BalanceResponse balance(RpcController controller,
1987 BalanceRequest request) throws ServiceException {
1988 return stub.balance(controller, request);
1989 }
1990
1991 @Override
1992 public SetBalancerRunningResponse setBalancerRunning(
1993 RpcController controller, SetBalancerRunningRequest request)
1994 throws ServiceException {
1995 return stub.setBalancerRunning(controller, request);
1996 }
1997
1998 @Override
1999 public RunCatalogScanResponse runCatalogScan(RpcController controller,
2000 RunCatalogScanRequest request) throws ServiceException {
2001 return stub.runCatalogScan(controller, request);
2002 }
2003
2004 @Override
2005 public EnableCatalogJanitorResponse enableCatalogJanitor(
2006 RpcController controller, EnableCatalogJanitorRequest request)
2007 throws ServiceException {
2008 return stub.enableCatalogJanitor(controller, request);
2009 }
2010
2011 @Override
2012 public IsCatalogJanitorEnabledResponse isCatalogJanitorEnabled(
2013 RpcController controller, IsCatalogJanitorEnabledRequest request)
2014 throws ServiceException {
2015 return stub.isCatalogJanitorEnabled(controller, request);
2016 }
2017
2018 @Override
2019 public CoprocessorServiceResponse execMasterService(
2020 RpcController controller, CoprocessorServiceRequest request)
2021 throws ServiceException {
2022 return stub.execMasterService(controller, request);
2023 }
2024
2025 @Override
2026 public SnapshotResponse snapshot(RpcController controller,
2027 SnapshotRequest request) throws ServiceException {
2028 return stub.snapshot(controller, request);
2029 }
2030
2031 @Override
2032 public GetCompletedSnapshotsResponse getCompletedSnapshots(
2033 RpcController controller, GetCompletedSnapshotsRequest request)
2034 throws ServiceException {
2035 return stub.getCompletedSnapshots(controller, request);
2036 }
2037
2038 @Override
2039 public DeleteSnapshotResponse deleteSnapshot(RpcController controller,
2040 DeleteSnapshotRequest request) throws ServiceException {
2041 return stub.deleteSnapshot(controller, request);
2042 }
2043
2044 @Override
2045 public IsSnapshotDoneResponse isSnapshotDone(RpcController controller,
2046 IsSnapshotDoneRequest request) throws ServiceException {
2047 return stub.isSnapshotDone(controller, request);
2048 }
2049
2050 @Override
2051 public RestoreSnapshotResponse restoreSnapshot(
2052 RpcController controller, RestoreSnapshotRequest request)
2053 throws ServiceException {
2054 return stub.restoreSnapshot(controller, request);
2055 }
2056
2057 @Override
2058 public IsRestoreSnapshotDoneResponse isRestoreSnapshotDone(
2059 RpcController controller, IsRestoreSnapshotDoneRequest request)
2060 throws ServiceException {
2061 return stub.isRestoreSnapshotDone(controller, request);
2062 }
2063
2064 @Override
2065 public IsMasterRunningResponse isMasterRunning(
2066 RpcController controller, IsMasterRunningRequest request)
2067 throws ServiceException {
2068 return stub.isMasterRunning(controller, request);
2069 }
2070
2071 @Override
2072 public ModifyNamespaceResponse modifyNamespace(RpcController controller,
2073 ModifyNamespaceRequest request)
2074 throws ServiceException {
2075 return stub.modifyNamespace(controller, request);
2076 }
2077
2078 @Override
2079 public CreateNamespaceResponse createNamespace(RpcController controller, CreateNamespaceRequest request) throws ServiceException {
2080 return stub.createNamespace(controller, request);
2081 }
2082
2083 @Override
2084 public DeleteNamespaceResponse deleteNamespace(RpcController controller, DeleteNamespaceRequest request) throws ServiceException {
2085 return stub.deleteNamespace(controller, request);
2086 }
2087
2088 @Override
2089 public GetNamespaceDescriptorResponse getNamespaceDescriptor(RpcController controller, GetNamespaceDescriptorRequest request) throws ServiceException {
2090 return stub.getNamespaceDescriptor(controller, request);
2091 }
2092
2093 @Override
2094 public ListNamespaceDescriptorsResponse listNamespaceDescriptors(RpcController controller, ListNamespaceDescriptorsRequest request) throws ServiceException {
2095 return stub.listNamespaceDescriptors(controller, request);
2096 }
2097
2098 @Override
2099 public ListTableDescriptorsByNamespaceResponse listTableDescriptorsByNamespace(RpcController controller, ListTableDescriptorsByNamespaceRequest request) throws ServiceException {
2100 return stub.listTableDescriptorsByNamespace(controller, request);
2101 }
2102
2103 @Override
2104 public ListTableNamesByNamespaceResponse listTableNamesByNamespace(RpcController controller,
2105 ListTableNamesByNamespaceRequest request) throws ServiceException {
2106 return stub.listTableNamesByNamespace(controller, request);
2107 }
2108
2109 @Override
2110 public void close() {
2111 release(this.mss);
2112 }
2113
2114 @Override
2115 public GetSchemaAlterStatusResponse getSchemaAlterStatus(
2116 RpcController controller, GetSchemaAlterStatusRequest request)
2117 throws ServiceException {
2118 return stub.getSchemaAlterStatus(controller, request);
2119 }
2120
2121 @Override
2122 public GetTableDescriptorsResponse getTableDescriptors(
2123 RpcController controller, GetTableDescriptorsRequest request)
2124 throws ServiceException {
2125 return stub.getTableDescriptors(controller, request);
2126 }
2127
2128 @Override
2129 public GetTableNamesResponse getTableNames(
2130 RpcController controller, GetTableNamesRequest request)
2131 throws ServiceException {
2132 return stub.getTableNames(controller, request);
2133 }
2134
2135 @Override
2136 public GetClusterStatusResponse getClusterStatus(
2137 RpcController controller, GetClusterStatusRequest request)
2138 throws ServiceException {
2139 return stub.getClusterStatus(controller, request);
2140 }
2141 };
2142 }
2143
2144
2145 private static void release(MasterServiceState mss) {
2146 if (mss != null && mss.connection != null) {
2147 ((HConnectionImplementation)mss.connection).releaseMaster(mss);
2148 }
2149 }
2150
2151 private boolean isKeepAliveMasterConnectedAndRunning(MasterServiceState mss) {
2152 if (mss.getStub() == null){
2153 return false;
2154 }
2155 try {
2156 return mss.isMasterRunning();
2157 } catch (UndeclaredThrowableException e) {
2158
2159
2160 LOG.info("Master connection is not running anymore", e.getUndeclaredThrowable());
2161 return false;
2162 } catch (ServiceException se) {
2163 LOG.warn("Checking master connection", se);
2164 return false;
2165 }
2166 }
2167
2168 void releaseMaster(MasterServiceState mss) {
2169 if (mss.getStub() == null) return;
2170 synchronized (masterAndZKLock) {
2171 --mss.userCount;
2172 if (mss.userCount <= 0) {
2173 mss.keepAliveUntil = System.currentTimeMillis() + keepAlive;
2174 }
2175 }
2176 }
2177
2178 private void closeMasterService(MasterServiceState mss) {
2179 if (mss.getStub() != null) {
2180 LOG.info("Closing master protocol: " + mss);
2181 mss.clearStub();
2182 }
2183 mss.userCount = 0;
2184 }
2185
2186
2187
2188
2189
2190 private void closeMaster() {
2191 synchronized (masterAndZKLock) {
2192 closeMasterService(masterServiceState);
2193 }
2194 }
2195
2196 void updateCachedLocation(HRegionInfo hri, HRegionLocation source,
2197 ServerName serverName, long seqNum) {
2198 HRegionLocation newHrl = new HRegionLocation(hri, serverName, seqNum);
2199 cacheLocation(hri.getTable(), source, newHrl);
2200 }
2201
2202
2203
2204
2205
2206
2207 void deleteCachedLocation(HRegionInfo hri, HRegionLocation source) {
2208 ConcurrentMap<byte[], HRegionLocation> tableLocations = getTableLocations(hri.getTable());
2209 tableLocations.remove(hri.getStartKey(), source);
2210 }
2211
2212 @Override
2213 public void deleteCachedRegionLocation(final HRegionLocation location) {
2214 if (location == null) {
2215 return;
2216 }
2217
2218 HRegionLocation removedLocation;
2219 TableName tableName = location.getRegionInfo().getTable();
2220 Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
2221 removedLocation = tableLocations.remove(location.getRegionInfo().getStartKey());
2222 if (LOG.isDebugEnabled() && removedLocation != null) {
2223 LOG.debug("Removed " +
2224 location.getRegionInfo().getRegionNameAsString() +
2225 " for tableName=" + tableName +
2226 " from cache");
2227 }
2228 }
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238 @Override
2239 public void updateCachedLocations(final TableName tableName, byte[] rowkey,
2240 final Object exception, final HRegionLocation source) {
2241 if (rowkey == null || tableName == null) {
2242 LOG.warn("Coding error, see method javadoc. row=" + (rowkey == null ? "null" : rowkey) +
2243 ", tableName=" + (tableName == null ? "null" : tableName));
2244 return;
2245 }
2246
2247 if (source == null || source.getServerName() == null){
2248
2249 return;
2250 }
2251
2252
2253 final HRegionLocation oldLocation = getCachedLocation(tableName, rowkey);
2254 if (oldLocation == null || !source.getServerName().equals(oldLocation.getServerName())) {
2255
2256
2257 return;
2258 }
2259
2260 HRegionInfo regionInfo = oldLocation.getRegionInfo();
2261 Throwable cause = findException(exception);
2262 if (cause != null) {
2263 if (cause instanceof RegionTooBusyException || cause instanceof RegionOpeningException) {
2264
2265 return;
2266 }
2267
2268 if (cause instanceof RegionMovedException) {
2269 RegionMovedException rme = (RegionMovedException) cause;
2270 if (LOG.isTraceEnabled()) {
2271 LOG.trace("Region " + regionInfo.getRegionNameAsString() + " moved to " +
2272 rme.getHostname() + ":" + rme.getPort() +
2273 " according to " + source.getHostnamePort());
2274 }
2275
2276
2277 updateCachedLocation(
2278 regionInfo, source, rme.getServerName(), rme.getLocationSeqNum());
2279 return;
2280 }
2281 }
2282
2283
2284
2285 deleteCachedLocation(regionInfo, source);
2286 }
2287
2288 @Override
2289 public void updateCachedLocations(final byte[] tableName, byte[] rowkey,
2290 final Object exception, final HRegionLocation source) {
2291 updateCachedLocations(TableName.valueOf(tableName), rowkey, exception, source);
2292 }
2293
2294 @Override
2295 @Deprecated
2296 public void processBatch(List<? extends Row> list,
2297 final TableName tableName,
2298 ExecutorService pool,
2299 Object[] results) throws IOException, InterruptedException {
2300
2301
2302
2303 if (results.length != list.size()) {
2304 throw new IllegalArgumentException(
2305 "argument results must be the same size as argument list");
2306 }
2307 processBatchCallback(list, tableName, pool, results, null);
2308 }
2309
2310 @Override
2311 @Deprecated
2312 public void processBatch(List<? extends Row> list,
2313 final byte[] tableName,
2314 ExecutorService pool,
2315 Object[] results) throws IOException, InterruptedException {
2316 processBatch(list, TableName.valueOf(tableName), pool, results);
2317 }
2318
2319
2320
2321
2322
2323
2324
2325
2326 @Override
2327 @Deprecated
2328 public <R> void processBatchCallback(
2329 List<? extends Row> list,
2330 TableName tableName,
2331 ExecutorService pool,
2332 Object[] results,
2333 Batch.Callback<R> callback)
2334 throws IOException, InterruptedException {
2335
2336
2337
2338 ObjectResultFiller<R> cb = new ObjectResultFiller<R>(results, callback);
2339 AsyncProcess<?> asyncProcess = createAsyncProcess(tableName, pool, cb, conf);
2340
2341
2342 asyncProcess.submitAll(list);
2343 asyncProcess.waitUntilDone();
2344
2345 if (asyncProcess.hasError()) {
2346 throw asyncProcess.getErrors();
2347 }
2348 }
2349
2350 @Override
2351 @Deprecated
2352 public <R> void processBatchCallback(
2353 List<? extends Row> list,
2354 byte[] tableName,
2355 ExecutorService pool,
2356 Object[] results,
2357 Batch.Callback<R> callback)
2358 throws IOException, InterruptedException {
2359 processBatchCallback(list, TableName.valueOf(tableName), pool, results, callback);
2360 }
2361
2362
2363 protected <R> AsyncProcess createAsyncProcess(TableName tableName, ExecutorService pool,
2364 AsyncProcess.AsyncProcessCallback<R> callback, Configuration conf) {
2365 return new AsyncProcess<R>(this, tableName, pool, callback, conf,
2366 RpcRetryingCallerFactory.instantiate(conf));
2367 }
2368
2369
2370
2371
2372
2373 private static class ObjectResultFiller<Res>
2374 implements AsyncProcess.AsyncProcessCallback<Res> {
2375
2376 private final Object[] results;
2377 private Batch.Callback<Res> callback;
2378
2379 ObjectResultFiller(Object[] results, Batch.Callback<Res> callback) {
2380 this.results = results;
2381 this.callback = callback;
2382 }
2383
2384 @Override
2385 public void success(int pos, byte[] region, Row row, Res result) {
2386 assert pos < results.length;
2387 results[pos] = result;
2388 if (callback != null) {
2389 callback.update(region, row.getRow(), result);
2390 }
2391 }
2392
2393 @Override
2394 public boolean failure(int pos, byte[] region, Row row, Throwable t) {
2395 assert pos < results.length;
2396 results[pos] = t;
2397
2398 return true;
2399 }
2400
2401 @Override
2402 public boolean retriableFailure(int originalIndex, Row row, byte[] region,
2403 Throwable exception) {
2404 return true;
2405 }
2406 }
2407
2408
2409
2410
2411
2412
2413 int getNumberOfCachedRegionLocations(final TableName tableName) {
2414 Map<byte[], HRegionLocation> tableLocs = this.cachedRegionLocations.get(tableName);
2415 if (tableLocs == null) {
2416 return 0;
2417 }
2418 return tableLocs.values().size();
2419 }
2420
2421
2422
2423
2424
2425
2426
2427
2428 boolean isRegionCached(TableName tableName, final byte[] row) {
2429 HRegionLocation location = getCachedLocation(tableName, row);
2430 return location != null;
2431 }
2432
2433 @Override
2434 public void setRegionCachePrefetch(final TableName tableName,
2435 final boolean enable) {
2436 if (!enable) {
2437 regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName.getName()));
2438 }
2439 else {
2440 regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName.getName()));
2441 }
2442 }
2443
2444 @Override
2445 public void setRegionCachePrefetch(final byte[] tableName,
2446 final boolean enable) {
2447 setRegionCachePrefetch(TableName.valueOf(tableName), enable);
2448 }
2449
2450 @Override
2451 public boolean getRegionCachePrefetch(TableName tableName) {
2452 return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName.getName()));
2453 }
2454
2455 @Override
2456 public boolean getRegionCachePrefetch(byte[] tableName) {
2457 return getRegionCachePrefetch(TableName.valueOf(tableName));
2458 }
2459
2460 @Override
2461 public void abort(final String msg, Throwable t) {
2462 if (t instanceof KeeperException.SessionExpiredException
2463 && keepAliveZookeeper != null) {
2464 synchronized (masterAndZKLock) {
2465 if (keepAliveZookeeper != null) {
2466 LOG.warn("This client just lost it's session with ZooKeeper," +
2467 " closing it." +
2468 " It will be recreated next time someone needs it", t);
2469 closeZooKeeperWatcher();
2470 }
2471 }
2472 } else {
2473 if (t != null) {
2474 LOG.fatal(msg, t);
2475 } else {
2476 LOG.fatal(msg);
2477 }
2478 this.aborted = true;
2479 close();
2480 this.closed = true;
2481 }
2482 }
2483
2484 @Override
2485 public boolean isClosed() {
2486 return this.closed;
2487 }
2488
2489 @Override
2490 public boolean isAborted(){
2491 return this.aborted;
2492 }
2493
2494 @Override
2495 public int getCurrentNrHRS() throws IOException {
2496 return this.registry.getCurrentNrHRS();
2497 }
2498
2499
2500
2501
2502 void incCount() {
2503 ++refCount;
2504 }
2505
2506
2507
2508
2509 void decCount() {
2510 if (refCount > 0) {
2511 --refCount;
2512 }
2513 }
2514
2515
2516
2517
2518
2519
2520 boolean isZeroReference() {
2521 return refCount == 0;
2522 }
2523
2524 void internalClose() {
2525 if (this.closed) {
2526 return;
2527 }
2528 delayedClosing.stop("Closing connection");
2529 closeMaster();
2530 shutdownBatchPool();
2531 this.closed = true;
2532 closeZooKeeperWatcher();
2533 this.stubs.clear();
2534 if (clusterStatusListener != null) {
2535 clusterStatusListener.close();
2536 }
2537 }
2538
2539 @Override
2540 public void close() {
2541 if (managed) {
2542 if (aborted) {
2543 HConnectionManager.deleteStaleConnection(this);
2544 } else {
2545 HConnectionManager.deleteConnection(this, false);
2546 }
2547 } else {
2548 internalClose();
2549 }
2550 }
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563 @Override
2564 protected void finalize() throws Throwable {
2565 super.finalize();
2566
2567 refCount = 1;
2568 close();
2569 }
2570
2571 @Override
2572 public HTableDescriptor[] listTables() throws IOException {
2573 MasterKeepAliveConnection master = getKeepAliveMasterService();
2574 try {
2575 GetTableDescriptorsRequest req =
2576 RequestConverter.buildGetTableDescriptorsRequest((List<TableName>)null);
2577 return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2578 } catch (ServiceException se) {
2579 throw ProtobufUtil.getRemoteException(se);
2580 } finally {
2581 master.close();
2582 }
2583 }
2584
2585 @Override
2586 public String[] getTableNames() throws IOException {
2587 TableName[] tableNames = listTableNames();
2588 String result[] = new String[tableNames.length];
2589 for (int i = 0; i < tableNames.length; i++) {
2590 result[i] = tableNames[i].getNameAsString();
2591 }
2592 return result;
2593 }
2594
2595 @Override
2596 public TableName[] listTableNames() throws IOException {
2597 MasterKeepAliveConnection master = getKeepAliveMasterService();
2598 try {
2599 return ProtobufUtil.getTableNameArray(master.getTableNames(null,
2600 GetTableNamesRequest.newBuilder().build())
2601 .getTableNamesList());
2602 } catch (ServiceException se) {
2603 throw ProtobufUtil.getRemoteException(se);
2604 } finally {
2605 master.close();
2606 }
2607 }
2608
2609 @Override
2610 public HTableDescriptor[] getHTableDescriptorsByTableName(
2611 List<TableName> tableNames) throws IOException {
2612 if (tableNames == null || tableNames.isEmpty()) return new HTableDescriptor[0];
2613 MasterKeepAliveConnection master = getKeepAliveMasterService();
2614 try {
2615 GetTableDescriptorsRequest req =
2616 RequestConverter.buildGetTableDescriptorsRequest(tableNames);
2617 return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2618 } catch (ServiceException se) {
2619 throw ProtobufUtil.getRemoteException(se);
2620 } finally {
2621 master.close();
2622 }
2623 }
2624
2625 @Override
2626 public HTableDescriptor[] getHTableDescriptors(
2627 List<String> names) throws IOException {
2628 List<TableName> tableNames = new ArrayList(names.size());
2629 for(String name : names) {
2630 tableNames.add(TableName.valueOf(name));
2631 }
2632
2633 return getHTableDescriptorsByTableName(tableNames);
2634 }
2635
2636 @Override
2637 public NonceGenerator getNonceGenerator() {
2638 return this.nonceGenerator;
2639 }
2640
2641
2642
2643
2644
2645
2646
2647
2648 @Override
2649 public HTableDescriptor getHTableDescriptor(final TableName tableName)
2650 throws IOException {
2651 if (tableName == null) return null;
2652 if (tableName.equals(TableName.META_TABLE_NAME)) {
2653 return HTableDescriptor.META_TABLEDESC;
2654 }
2655 MasterKeepAliveConnection master = getKeepAliveMasterService();
2656 GetTableDescriptorsResponse htds;
2657 try {
2658 GetTableDescriptorsRequest req =
2659 RequestConverter.buildGetTableDescriptorsRequest(tableName);
2660 htds = master.getTableDescriptors(null, req);
2661 } catch (ServiceException se) {
2662 throw ProtobufUtil.getRemoteException(se);
2663 } finally {
2664 master.close();
2665 }
2666 if (!htds.getTableSchemaList().isEmpty()) {
2667 return HTableDescriptor.convert(htds.getTableSchemaList().get(0));
2668 }
2669 throw new TableNotFoundException(tableName.getNameAsString());
2670 }
2671
2672 @Override
2673 public HTableDescriptor getHTableDescriptor(final byte[] tableName)
2674 throws IOException {
2675 return getHTableDescriptor(TableName.valueOf(tableName));
2676 }
2677 }
2678
2679
2680
2681
2682 static class ServerErrorTracker {
2683
2684 private final ConcurrentMap<HRegionLocation, ServerErrors> errorsByServer =
2685 new ConcurrentHashMap<HRegionLocation, ServerErrors>();
2686 private final long canRetryUntil;
2687 private final int maxRetries;
2688 private final String startTrackingTime;
2689
2690 public ServerErrorTracker(long timeout, int maxRetries) {
2691 this.maxRetries = maxRetries;
2692 this.canRetryUntil = EnvironmentEdgeManager.currentTimeMillis() + timeout;
2693 this.startTrackingTime = new Date().toString();
2694 }
2695
2696
2697
2698
2699 boolean canRetryMore(int numRetry) {
2700
2701 return numRetry < maxRetries || (maxRetries > 1 &&
2702 EnvironmentEdgeManager.currentTimeMillis() < this.canRetryUntil);
2703 }
2704
2705
2706
2707
2708
2709
2710
2711
2712 long calculateBackoffTime(HRegionLocation server, long basePause) {
2713 long result;
2714 ServerErrors errorStats = errorsByServer.get(server);
2715 if (errorStats != null) {
2716 result = ConnectionUtils.getPauseTime(basePause, errorStats.retries.get());
2717 } else {
2718 result = 0;
2719 }
2720 return result;
2721 }
2722
2723
2724
2725
2726
2727
2728 void reportServerError(HRegionLocation server) {
2729 ServerErrors errors = errorsByServer.get(server);
2730 if (errors != null) {
2731 errors.addError();
2732 } else {
2733 errors = errorsByServer.putIfAbsent(server, new ServerErrors());
2734 if (errors != null){
2735 errors.addError();
2736 }
2737 }
2738 }
2739
2740 String getStartTrackingTime() {
2741 return startTrackingTime;
2742 }
2743
2744
2745
2746
2747 private static class ServerErrors {
2748 public final AtomicInteger retries = new AtomicInteger(0);
2749
2750 public void addError() {
2751 retries.incrementAndGet();
2752 }
2753 }
2754 }
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764 public static Throwable findException(Object exception) {
2765 if (exception == null || !(exception instanceof Throwable)) {
2766 return null;
2767 }
2768 Throwable cur = (Throwable) exception;
2769 while (cur != null) {
2770 if (cur instanceof RegionMovedException || cur instanceof RegionOpeningException
2771 || cur instanceof RegionTooBusyException) {
2772 return cur;
2773 }
2774 if (cur instanceof RemoteException) {
2775 RemoteException re = (RemoteException) cur;
2776 cur = re.unwrapRemoteException(
2777 RegionOpeningException.class, RegionMovedException.class,
2778 RegionTooBusyException.class);
2779 if (cur == null) {
2780 cur = re.unwrapRemoteException();
2781 }
2782
2783
2784
2785 if (cur == re) {
2786 return null;
2787 }
2788 } else {
2789 cur = cur.getCause();
2790 }
2791 }
2792
2793 return null;
2794 }
2795
2796
2797
2798
2799
2800
2801
2802
2803 public static void setServerSideHConnectionRetries(final Configuration c, final String sn,
2804 final Log log) {
2805 int hcRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2806 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2807
2808
2809 int serversideMultiplier = c.getInt("hbase.client.serverside.retries.multiplier", 10);
2810 int retries = hcRetries * serversideMultiplier;
2811 c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, retries);
2812 log.debug(sn + " HConnection server-to-server retries=" + retries);
2813 }
2814 }