1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.io.Closeable;
22 import java.io.IOException;
23 import java.lang.reflect.Constructor;
24 import java.lang.reflect.UndeclaredThrowableException;
25 import java.net.SocketException;
26 import java.util.ArrayList;
27 import java.util.Date;
28 import java.util.HashSet;
29 import java.util.LinkedHashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Map.Entry;
33 import java.util.NavigableMap;
34 import java.util.Set;
35 import java.util.concurrent.ConcurrentHashMap;
36 import java.util.concurrent.ConcurrentMap;
37 import java.util.concurrent.ConcurrentSkipListMap;
38 import java.util.concurrent.ConcurrentSkipListSet;
39 import java.util.concurrent.CopyOnWriteArraySet;
40 import java.util.concurrent.ExecutorService;
41 import java.util.concurrent.LinkedBlockingQueue;
42 import java.util.concurrent.ThreadPoolExecutor;
43 import java.util.concurrent.TimeUnit;
44 import java.util.concurrent.atomic.AtomicBoolean;
45 import java.util.concurrent.atomic.AtomicInteger;
46
47 import org.apache.commons.logging.Log;
48 import org.apache.commons.logging.LogFactory;
49 import org.apache.hadoop.classification.InterfaceAudience;
50 import org.apache.hadoop.classification.InterfaceStability;
51 import org.apache.hadoop.conf.Configuration;
52 import org.apache.hadoop.hbase.Chore;
53 import org.apache.hadoop.hbase.HBaseConfiguration;
54 import org.apache.hadoop.hbase.HConstants;
55 import org.apache.hadoop.hbase.HRegionInfo;
56 import org.apache.hadoop.hbase.HRegionLocation;
57 import org.apache.hadoop.hbase.HTableDescriptor;
58 import org.apache.hadoop.hbase.MasterNotRunningException;
59 import org.apache.hadoop.hbase.RegionTooBusyException;
60 import org.apache.hadoop.hbase.ServerName;
61 import org.apache.hadoop.hbase.Stoppable;
62 import org.apache.hadoop.hbase.TableName;
63 import org.apache.hadoop.hbase.TableNotEnabledException;
64 import org.apache.hadoop.hbase.TableNotFoundException;
65 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
66 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
67 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
68 import org.apache.hadoop.hbase.client.coprocessor.Batch;
69 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
70 import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
71 import org.apache.hadoop.hbase.ipc.RpcClient;
72 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73 import org.apache.hadoop.hbase.protobuf.RequestConverter;
74 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
75 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
76 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
77 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
78 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnResponse;
79 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionResponse;
80 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceResponse;
81 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableResponse;
82 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnResponse;
83 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotResponse;
84 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableResponse;
85 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse;
86 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse;
87 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
88 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse;
89 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsResponse;
90 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
91 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
92 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest;
93 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse;
94 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
95 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse;
96 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
97 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
98 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceResponse;
99 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnResponse;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableResponse;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionResponse;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionResponse;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterResponse;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionResponse;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.*;
115 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
116 import org.apache.hadoop.hbase.security.User;
117 import org.apache.hadoop.hbase.security.UserProvider;
118 import org.apache.hadoop.hbase.util.Bytes;
119 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
120 import org.apache.hadoop.hbase.util.Threads;
121 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
122 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
123 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
124 import org.apache.hadoop.ipc.RemoteException;
125 import org.apache.zookeeper.KeeperException;
126
127 import com.google.protobuf.BlockingRpcChannel;
128 import com.google.protobuf.RpcController;
129 import com.google.protobuf.ServiceException;
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192 @SuppressWarnings("serial")
193 @InterfaceAudience.Public
194 @InterfaceStability.Evolving
195 public class HConnectionManager {
196 static final Log LOG = LogFactory.getLog(HConnectionManager.class);
197
198 public static final String RETRIES_BY_SERVER_KEY = "hbase.client.retries.by.server";
199
200
201
202
203 static final Map<HConnectionKey, HConnectionImplementation> CONNECTION_INSTANCES;
204
205 public static final int MAX_CACHED_CONNECTION_INSTANCES;
206
207 static {
208
209
210
211
212 MAX_CACHED_CONNECTION_INSTANCES = HBaseConfiguration.create().getInt(
213 HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1;
214 CONNECTION_INSTANCES = new LinkedHashMap<HConnectionKey, HConnectionImplementation>(
215 (int) (MAX_CACHED_CONNECTION_INSTANCES / 0.75F) + 1, 0.75F, true) {
216 @Override
217 protected boolean removeEldestEntry(
218 Map.Entry<HConnectionKey, HConnectionImplementation> eldest) {
219 return size() > MAX_CACHED_CONNECTION_INSTANCES;
220 }
221 };
222 }
223
224
225
226
227 private HConnectionManager() {
228 super();
229 }
230
231
232
233
234
235
236
237
238
239
240 @Deprecated
241 public static HConnection getConnection(final Configuration conf)
242 throws IOException {
243 HConnectionKey connectionKey = new HConnectionKey(conf);
244 synchronized (CONNECTION_INSTANCES) {
245 HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);
246 if (connection == null) {
247 connection = (HConnectionImplementation)createConnection(conf, true);
248 CONNECTION_INSTANCES.put(connectionKey, connection);
249 } else if (connection.isClosed()) {
250 HConnectionManager.deleteConnection(connectionKey, true);
251 connection = (HConnectionImplementation)createConnection(conf, true);
252 CONNECTION_INSTANCES.put(connectionKey, connection);
253 }
254 connection.incCount();
255 return connection;
256 }
257 }
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279 public static HConnection createConnection(Configuration conf)
280 throws IOException {
281 UserProvider provider = UserProvider.instantiate(conf);
282 return createConnection(conf, false, null, provider.getCurrent());
283 }
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305 public static HConnection createConnection(Configuration conf, ExecutorService pool)
306 throws IOException {
307 UserProvider provider = UserProvider.instantiate(conf);
308 return createConnection(conf, false, pool, provider.getCurrent());
309 }
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331 public static HConnection createConnection(Configuration conf, User user)
332 throws IOException {
333 return createConnection(conf, false, null, user);
334 }
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357 public static HConnection createConnection(Configuration conf, ExecutorService pool, User user)
358 throws IOException {
359 return createConnection(conf, false, pool, user);
360 }
361
362 @Deprecated
363 static HConnection createConnection(final Configuration conf, final boolean managed)
364 throws IOException {
365 UserProvider provider = UserProvider.instantiate(conf);
366 return createConnection(conf, managed, null, provider.getCurrent());
367 }
368
369 @Deprecated
370 static HConnection createConnection(final Configuration conf, final boolean managed,
371 final ExecutorService pool, final User user)
372 throws IOException {
373 String className = conf.get("hbase.client.connection.impl",
374 HConnectionManager.HConnectionImplementation.class.getName());
375 Class<?> clazz = null;
376 try {
377 clazz = Class.forName(className);
378 } catch (ClassNotFoundException e) {
379 throw new IOException(e);
380 }
381 try {
382
383 Constructor<?> constructor =
384 clazz.getDeclaredConstructor(Configuration.class,
385 boolean.class, ExecutorService.class, User.class);
386 constructor.setAccessible(true);
387 return (HConnection) constructor.newInstance(conf, managed, pool, user);
388 } catch (Exception e) {
389 throw new IOException(e);
390 }
391 }
392
393
394
395
396
397
398
399
400
401 public static void deleteConnection(Configuration conf) {
402 deleteConnection(new HConnectionKey(conf), false);
403 }
404
405
406
407
408
409
410
411
412 public static void deleteStaleConnection(HConnection connection) {
413 deleteConnection(connection, true);
414 }
415
416
417
418
419
420
421
422 public static void deleteAllConnections(boolean staleConnection) {
423 synchronized (CONNECTION_INSTANCES) {
424 Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
425 connectionKeys.addAll(CONNECTION_INSTANCES.keySet());
426 for (HConnectionKey connectionKey : connectionKeys) {
427 deleteConnection(connectionKey, staleConnection);
428 }
429 CONNECTION_INSTANCES.clear();
430 }
431 }
432
433
434
435
436
437 @Deprecated
438 public static void deleteAllConnections() {
439 deleteAllConnections(false);
440 }
441
442
443 @Deprecated
444 private static void deleteConnection(HConnection connection, boolean staleConnection) {
445 synchronized (CONNECTION_INSTANCES) {
446 for (Entry<HConnectionKey, HConnectionImplementation> e: CONNECTION_INSTANCES.entrySet()) {
447 if (e.getValue() == connection) {
448 deleteConnection(e.getKey(), staleConnection);
449 break;
450 }
451 }
452 }
453 }
454
455 @Deprecated
456 private static void deleteConnection(HConnectionKey connectionKey, boolean staleConnection) {
457 synchronized (CONNECTION_INSTANCES) {
458 HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);
459 if (connection != null) {
460 connection.decCount();
461 if (connection.isZeroReference() || staleConnection) {
462 CONNECTION_INSTANCES.remove(connectionKey);
463 connection.internalClose();
464 }
465 } else {
466 LOG.error("Connection not found in the list, can't delete it "+
467 "(connection key=" + connectionKey + "). May be the key was modified?", new Exception());
468 }
469 }
470 }
471
472
473
474
475
476
477
478 static int getCachedRegionCount(Configuration conf, final TableName tableName)
479 throws IOException {
480 return execute(new HConnectable<Integer>(conf) {
481 @Override
482 public Integer connect(HConnection connection) {
483 return ((HConnectionImplementation)connection).getNumberOfCachedRegionLocations(tableName);
484 }
485 });
486 }
487
488
489
490
491
492
493
494 static boolean isRegionCached(Configuration conf,
495 final TableName tableName,
496 final byte[] row)
497 throws IOException {
498 return execute(new HConnectable<Boolean>(conf) {
499 @Override
500 public Boolean connect(HConnection connection) {
501 return ((HConnectionImplementation) connection).isRegionCached(tableName, row);
502 }
503 });
504 }
505
506
507
508
509
510
511
512
513
514
515
516 @InterfaceAudience.Private
517 public static <T> T execute(HConnectable<T> connectable) throws IOException {
518 if (connectable == null || connectable.conf == null) {
519 return null;
520 }
521 Configuration conf = connectable.conf;
522 HConnection connection = HConnectionManager.getConnection(conf);
523 boolean connectSucceeded = false;
524 try {
525 T returnValue = connectable.connect(connection);
526 connectSucceeded = true;
527 return returnValue;
528 } finally {
529 try {
530 connection.close();
531 } catch (Exception e) {
532 if (connectSucceeded) {
533 throw new IOException("The connection to " + connection
534 + " could not be deleted.", e);
535 }
536 }
537 }
538 }
539
540
541 @edu.umd.cs.findbugs.annotations.SuppressWarnings(
542 value="AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION",
543 justification="Access to the conncurrent hash map is under a lock so should be fine.")
544 static class HConnectionImplementation implements HConnection, Closeable {
545 static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);
546 private final long pause;
547 private final int numTries;
548 final int rpcTimeout;
549 private final int prefetchRegionLimit;
550
551 private volatile boolean closed;
552 private volatile boolean aborted;
553
554
555 ClusterStatusListener clusterStatusListener;
556
557 private final Object userRegionLock = new Object();
558
559
560
561
562
563
564 private final Object masterAndZKLock = new Object();
565
566 private long keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
567 private final DelayedClosing delayedClosing =
568 DelayedClosing.createAndStart(this);
569
570
571
572 private volatile ExecutorService batchPool = null;
573 private volatile boolean cleanupPool = false;
574
575 private final Configuration conf;
576
577
578 private RpcClient rpcClient;
579
580
581
582
583 private final ConcurrentMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>
584 cachedRegionLocations =
585 new ConcurrentHashMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>();
586
587
588
589
590
591
592 private final Set<ServerName> cachedServers = new ConcurrentSkipListSet<ServerName>();
593
594
595
596 private final Set<Integer> regionCachePrefetchDisabledTables =
597 new CopyOnWriteArraySet<Integer>();
598
599 private int refCount;
600
601
602 private boolean managed;
603
604 private User user;
605
606
607
608
609 Registry registry;
610
611 HConnectionImplementation(Configuration conf, boolean managed) throws IOException {
612 this(conf, managed, null, null);
613 }
614
615
616
617
618
619
620
621
622
623
624
625
626 HConnectionImplementation(Configuration conf, boolean managed,
627 ExecutorService pool, User user) throws IOException {
628 this(conf);
629 this.user = user;
630 this.batchPool = pool;
631 this.managed = managed;
632 this.registry = setupRegistry();
633 retrieveClusterId();
634
635 this.rpcClient = new RpcClient(this.conf, this.clusterId);
636
637
638 boolean shouldListen = conf.getBoolean(HConstants.STATUS_PUBLISHED,
639 HConstants.STATUS_PUBLISHED_DEFAULT);
640 Class<? extends ClusterStatusListener.Listener> listenerClass =
641 conf.getClass(ClusterStatusListener.STATUS_LISTENER_CLASS,
642 ClusterStatusListener.DEFAULT_STATUS_LISTENER_CLASS,
643 ClusterStatusListener.Listener.class);
644 if (shouldListen) {
645 if (listenerClass == null) {
646 LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
647 ClusterStatusListener.STATUS_LISTENER_CLASS + " is not set - not listening status");
648 } else {
649 clusterStatusListener = new ClusterStatusListener(
650 new ClusterStatusListener.DeadServerHandler() {
651 @Override
652 public void newDead(ServerName sn) {
653 clearCaches(sn);
654 rpcClient.cancelConnections(sn.getHostname(), sn.getPort(),
655 new SocketException(sn.getServerName() +
656 " is dead: closing its connection."));
657 }
658 }, conf, listenerClass);
659 }
660 }
661 }
662
663
664
665
666
667 protected HConnectionImplementation(Configuration conf) {
668 this.conf = conf;
669 this.closed = false;
670 this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
671 HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
672 this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
673 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
674 this.rpcTimeout = conf.getInt(
675 HConstants.HBASE_RPC_TIMEOUT_KEY,
676 HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
677 this.prefetchRegionLimit = conf.getInt(
678 HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
679 HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
680 }
681
682 @Override
683 public HTableInterface getTable(String tableName) throws IOException {
684 return getTable(TableName.valueOf(tableName));
685 }
686
687 @Override
688 public HTableInterface getTable(byte[] tableName) throws IOException {
689 return getTable(TableName.valueOf(tableName));
690 }
691
692 @Override
693 public HTableInterface getTable(TableName tableName) throws IOException {
694 return getTable(tableName, getBatchPool());
695 }
696
697 @Override
698 public HTableInterface getTable(String tableName, ExecutorService pool) throws IOException {
699 return getTable(TableName.valueOf(tableName), pool);
700 }
701
702 @Override
703 public HTableInterface getTable(byte[] tableName, ExecutorService pool) throws IOException {
704 return getTable(TableName.valueOf(tableName), pool);
705 }
706
707 @Override
708 public HTableInterface getTable(TableName tableName, ExecutorService pool) throws IOException {
709 if (managed) {
710 throw new IOException("The connection has to be unmanaged.");
711 }
712 return new HTable(tableName, this, pool);
713 }
714
715 private ExecutorService getBatchPool() {
716 if (batchPool == null) {
717
718 synchronized (this) {
719 if (batchPool == null) {
720 int maxThreads = conf.getInt("hbase.hconnection.threads.max", 256);
721 int coreThreads = conf.getInt("hbase.hconnection.threads.core", 256);
722 if (maxThreads == 0) {
723 maxThreads = Runtime.getRuntime().availableProcessors() * 8;
724 }
725 if (coreThreads == 0) {
726 coreThreads = Runtime.getRuntime().availableProcessors() * 8;
727 }
728 long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60);
729 LinkedBlockingQueue<Runnable> workQueue =
730 new LinkedBlockingQueue<Runnable>(maxThreads *
731 conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
732 HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
733 ThreadPoolExecutor tpe = new ThreadPoolExecutor(
734 coreThreads,
735 maxThreads,
736 keepAliveTime,
737 TimeUnit.SECONDS,
738 workQueue,
739 Threads.newDaemonThreadFactory(toString() + "-shared-"));
740 tpe.allowCoreThreadTimeOut(true);
741 this.batchPool = tpe;
742 }
743 this.cleanupPool = true;
744 }
745 }
746 return this.batchPool;
747 }
748
749 protected ExecutorService getCurrentBatchPool() {
750 return batchPool;
751 }
752
753 private void shutdownBatchPool() {
754 if (this.cleanupPool && this.batchPool != null && !this.batchPool.isShutdown()) {
755 this.batchPool.shutdown();
756 try {
757 if (!this.batchPool.awaitTermination(10, TimeUnit.SECONDS)) {
758 this.batchPool.shutdownNow();
759 }
760 } catch (InterruptedException e) {
761 this.batchPool.shutdownNow();
762 }
763 }
764 }
765
766
767
768
769
770 private Registry setupRegistry() throws IOException {
771 String registryClass = this.conf.get("hbase.client.registry.impl",
772 ZooKeeperRegistry.class.getName());
773 Registry registry = null;
774 try {
775 registry = (Registry)Class.forName(registryClass).newInstance();
776 } catch (Throwable t) {
777 throw new IOException(t);
778 }
779 registry.init(this);
780 return registry;
781 }
782
783
784
785
786
787
788 RpcClient setRpcClient(final RpcClient rpcClient) {
789 RpcClient oldRpcClient = this.rpcClient;
790 this.rpcClient = rpcClient;
791 return oldRpcClient;
792 }
793
794
795
796
797
798 public String toString(){
799 return "hconnection-0x" + Integer.toHexString(hashCode());
800 }
801
802 protected String clusterId = null;
803
804 void retrieveClusterId() {
805 if (clusterId != null) return;
806 this.clusterId = this.registry.getClusterId();
807 if (clusterId == null) {
808 clusterId = HConstants.CLUSTER_ID_DEFAULT;
809 LOG.debug("clusterid came back null, using default " + clusterId);
810 }
811 }
812
813 @Override
814 public Configuration getConfiguration() {
815 return this.conf;
816 }
817
818 private void checkIfBaseNodeAvailable(ZooKeeperWatcher zkw)
819 throws MasterNotRunningException {
820 String errorMsg;
821 try {
822 if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {
823 errorMsg = "The node " + zkw.baseZNode+" is not in ZooKeeper. "
824 + "It should have been written by the master. "
825 + "Check the value configured in 'zookeeper.znode.parent'. "
826 + "There could be a mismatch with the one configured in the master.";
827 LOG.error(errorMsg);
828 throw new MasterNotRunningException(errorMsg);
829 }
830 } catch (KeeperException e) {
831 errorMsg = "Can't get connection to ZooKeeper: " + e.getMessage();
832 LOG.error(errorMsg);
833 throw new MasterNotRunningException(errorMsg, e);
834 }
835 }
836
837
838
839
840
841
842 @Override
843 public boolean isMasterRunning()
844 throws MasterNotRunningException, ZooKeeperConnectionException {
845
846
847
848 MasterKeepAliveConnection m = getKeepAliveMasterService();
849 m.close();
850 return true;
851 }
852
853 @Override
854 public HRegionLocation getRegionLocation(final TableName tableName,
855 final byte [] row, boolean reload)
856 throws IOException {
857 return reload? relocateRegion(tableName, row): locateRegion(tableName, row);
858 }
859
860 @Override
861 public HRegionLocation getRegionLocation(final byte[] tableName,
862 final byte [] row, boolean reload)
863 throws IOException {
864 return getRegionLocation(TableName.valueOf(tableName), row, reload);
865 }
866
867 @Override
868 public boolean isTableEnabled(TableName tableName) throws IOException {
869 return this.registry.isTableOnlineState(tableName, true);
870 }
871
872 @Override
873 public boolean isTableEnabled(byte[] tableName) throws IOException {
874 return isTableEnabled(TableName.valueOf(tableName));
875 }
876
877 @Override
878 public boolean isTableDisabled(TableName tableName) throws IOException {
879 return this.registry.isTableOnlineState(tableName, false);
880 }
881
882 @Override
883 public boolean isTableDisabled(byte[] tableName) throws IOException {
884 return isTableDisabled(TableName.valueOf(tableName));
885 }
886
887 @Override
888 public boolean isTableAvailable(final TableName tableName) throws IOException {
889 final AtomicBoolean available = new AtomicBoolean(true);
890 final AtomicInteger regionCount = new AtomicInteger(0);
891 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
892 @Override
893 public boolean processRow(Result row) throws IOException {
894 HRegionInfo info = MetaScanner.getHRegionInfo(row);
895 if (info != null && !info.isSplitParent()) {
896 if (tableName.equals(info.getTable())) {
897 ServerName server = HRegionInfo.getServerName(row);
898 if (server == null) {
899 available.set(false);
900 return false;
901 }
902 regionCount.incrementAndGet();
903 } else if (tableName.compareTo(info.getTable()) < 0) {
904
905 return false;
906 }
907 }
908 return true;
909 }
910 };
911 MetaScanner.metaScan(conf, this, visitor, tableName);
912 return available.get() && (regionCount.get() > 0);
913 }
914
915 @Override
916 public boolean isTableAvailable(final byte[] tableName) throws IOException {
917 return isTableAvailable(TableName.valueOf(tableName));
918 }
919
920 @Override
921 public boolean isTableAvailable(final TableName tableName, final byte[][] splitKeys)
922 throws IOException {
923 final AtomicBoolean available = new AtomicBoolean(true);
924 final AtomicInteger regionCount = new AtomicInteger(0);
925 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
926 @Override
927 public boolean processRow(Result row) throws IOException {
928 HRegionInfo info = MetaScanner.getHRegionInfo(row);
929 if (info != null && !info.isSplitParent()) {
930 if (tableName.equals(info.getTable())) {
931 ServerName server = HRegionInfo.getServerName(row);
932 if (server == null) {
933 available.set(false);
934 return false;
935 }
936 if (!Bytes.equals(info.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
937 for (byte[] splitKey : splitKeys) {
938
939 if (Bytes.equals(info.getStartKey(), splitKey)) {
940 regionCount.incrementAndGet();
941 break;
942 }
943 }
944 } else {
945
946 regionCount.incrementAndGet();
947 }
948 } else if (tableName.compareTo(info.getTable()) < 0) {
949
950 return false;
951 }
952 }
953 return true;
954 }
955 };
956 MetaScanner.metaScan(conf, this, visitor, tableName);
957
958 return available.get() && (regionCount.get() == splitKeys.length + 1);
959 }
960
961 @Override
962 public boolean isTableAvailable(final byte[] tableName, final byte[][] splitKeys)
963 throws IOException {
964 return isTableAvailable(TableName.valueOf(tableName), splitKeys);
965 }
966
967 @Override
968 public HRegionLocation locateRegion(final byte[] regionName) throws IOException {
969 return locateRegion(HRegionInfo.getTable(regionName),
970 HRegionInfo.getStartKey(regionName), false, true);
971 }
972
973 @Override
974 public boolean isDeadServer(ServerName sn) {
975 if (clusterStatusListener == null) {
976 return false;
977 } else {
978 return clusterStatusListener.isDeadServer(sn);
979 }
980 }
981
982 @Override
983 public List<HRegionLocation> locateRegions(final TableName tableName)
984 throws IOException {
985 return locateRegions (tableName, false, true);
986 }
987
988 @Override
989 public List<HRegionLocation> locateRegions(final byte[] tableName)
990 throws IOException {
991 return locateRegions(TableName.valueOf(tableName));
992 }
993
994 @Override
995 public List<HRegionLocation> locateRegions(final TableName tableName,
996 final boolean useCache, final boolean offlined) throws IOException {
997 NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, this,
998 tableName, offlined);
999 final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();
1000 for (HRegionInfo regionInfo : regions.keySet()) {
1001 locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true));
1002 }
1003 return locations;
1004 }
1005
1006 @Override
1007 public List<HRegionLocation> locateRegions(final byte[] tableName,
1008 final boolean useCache, final boolean offlined) throws IOException {
1009 return locateRegions(TableName.valueOf(tableName), useCache, offlined);
1010 }
1011
1012 @Override
1013 public HRegionLocation locateRegion(final TableName tableName,
1014 final byte [] row)
1015 throws IOException{
1016 return locateRegion(tableName, row, true, true);
1017 }
1018
1019 @Override
1020 public HRegionLocation locateRegion(final byte[] tableName,
1021 final byte [] row)
1022 throws IOException{
1023 return locateRegion(TableName.valueOf(tableName), row);
1024 }
1025
1026 @Override
1027 public HRegionLocation relocateRegion(final TableName tableName,
1028 final byte [] row) throws IOException{
1029
1030
1031
1032 if (isTableDisabled(tableName)) {
1033 throw new TableNotEnabledException(tableName.getNameAsString() + " is disabled.");
1034 }
1035
1036 return locateRegion(tableName, row, false, true);
1037 }
1038
1039 @Override
1040 public HRegionLocation relocateRegion(final byte[] tableName,
1041 final byte [] row) throws IOException {
1042 return relocateRegion(TableName.valueOf(tableName), row);
1043 }
1044
1045
1046 private HRegionLocation locateRegion(final TableName tableName,
1047 final byte [] row, boolean useCache, boolean retry)
1048 throws IOException {
1049 if (this.closed) throw new IOException(toString() + " closed");
1050 if (tableName== null || tableName.getName().length == 0) {
1051 throw new IllegalArgumentException(
1052 "table name cannot be null or zero length");
1053 }
1054
1055 if (tableName.equals(TableName.META_TABLE_NAME)) {
1056 return this.registry.getMetaRegionLocation();
1057 } else {
1058
1059 return locateRegionInMeta(TableName.META_TABLE_NAME, tableName, row,
1060 useCache, userRegionLock, retry);
1061 }
1062 }
1063
1064
1065
1066
1067
1068
1069 private void prefetchRegionCache(final TableName tableName,
1070 final byte[] row) {
1071
1072
1073 MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
1074 public boolean processRow(Result result) throws IOException {
1075 try {
1076 HRegionInfo regionInfo = MetaScanner.getHRegionInfo(result);
1077 if (regionInfo == null) {
1078 return true;
1079 }
1080
1081
1082 if (!regionInfo.getTable().equals(tableName)) {
1083 return false;
1084 }
1085 if (regionInfo.isOffline()) {
1086
1087 return true;
1088 }
1089
1090 ServerName serverName = HRegionInfo.getServerName(result);
1091 if (serverName == null) {
1092 return true;
1093 }
1094
1095 long seqNum = HRegionInfo.getSeqNumDuringOpen(result);
1096 HRegionLocation loc = new HRegionLocation(regionInfo, serverName, seqNum);
1097
1098 cacheLocation(tableName, null, loc);
1099 return true;
1100 } catch (RuntimeException e) {
1101 throw new IOException(e);
1102 }
1103 }
1104 };
1105 try {
1106
1107 MetaScanner.metaScan(conf, this, visitor, tableName, row,
1108 this.prefetchRegionLimit, TableName.META_TABLE_NAME);
1109 } catch (IOException e) {
1110 LOG.warn("Encountered problems when prefetch hbase:meta table: ", e);
1111 }
1112 }
1113
1114
1115
1116
1117
1118 private HRegionLocation locateRegionInMeta(final TableName parentTable,
1119 final TableName tableName, final byte [] row, boolean useCache,
1120 Object regionLockObject, boolean retry)
1121 throws IOException {
1122 HRegionLocation location;
1123
1124
1125 if (useCache) {
1126 location = getCachedLocation(tableName, row);
1127 if (location != null) {
1128 return location;
1129 }
1130 }
1131 int localNumRetries = retry ? numTries : 1;
1132
1133
1134
1135 byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
1136 HConstants.NINES, false);
1137 for (int tries = 0; true; tries++) {
1138 if (tries >= localNumRetries) {
1139 throw new NoServerForRegionException("Unable to find region for "
1140 + Bytes.toStringBinary(row) + " after " + numTries + " tries.");
1141 }
1142
1143 HRegionLocation metaLocation = null;
1144 try {
1145
1146 metaLocation = locateRegion(parentTable, metaKey, true, false);
1147
1148 if (metaLocation == null) continue;
1149 ClientService.BlockingInterface service = getClient(metaLocation.getServerName());
1150
1151 Result regionInfoRow;
1152
1153
1154
1155 if (useCache) {
1156 if (TableName.META_TABLE_NAME.equals(parentTable) &&
1157 getRegionCachePrefetch(tableName)) {
1158 synchronized (regionLockObject) {
1159
1160
1161 location = getCachedLocation(tableName, row);
1162 if (location != null) {
1163 return location;
1164 }
1165
1166
1167 prefetchRegionCache(tableName, row);
1168 }
1169 }
1170 location = getCachedLocation(tableName, row);
1171 if (location != null) {
1172 return location;
1173 }
1174 } else {
1175
1176
1177 forceDeleteCachedLocation(tableName, row);
1178 }
1179
1180
1181 regionInfoRow = ProtobufUtil.getRowOrBefore(service,
1182 metaLocation.getRegionInfo().getRegionName(), metaKey,
1183 HConstants.CATALOG_FAMILY);
1184
1185 if (regionInfoRow == null) {
1186 throw new TableNotFoundException(tableName);
1187 }
1188
1189
1190 HRegionInfo regionInfo = MetaScanner.getHRegionInfo(regionInfoRow);
1191 if (regionInfo == null) {
1192 throw new IOException("HRegionInfo was null or empty in " +
1193 parentTable + ", row=" + regionInfoRow);
1194 }
1195
1196
1197 if (!regionInfo.getTable().equals(tableName)) {
1198 throw new TableNotFoundException(
1199 "Table '" + tableName + "' was not found, got: " +
1200 regionInfo.getTable() + ".");
1201 }
1202 if (regionInfo.isSplit()) {
1203 throw new RegionOfflineException("the only available region for" +
1204 " the required row is a split parent," +
1205 " the daughters should be online soon: " +
1206 regionInfo.getRegionNameAsString());
1207 }
1208 if (regionInfo.isOffline()) {
1209 throw new RegionOfflineException("the region is offline, could" +
1210 " be caused by a disable table call: " +
1211 regionInfo.getRegionNameAsString());
1212 }
1213
1214 ServerName serverName = HRegionInfo.getServerName(regionInfoRow);
1215 if (serverName == null) {
1216 throw new NoServerForRegionException("No serverName " +
1217 "in " + parentTable + " for " +
1218 regionInfo.getRegionNameAsString() + " containing " +
1219 Bytes.toStringBinary(row));
1220 }
1221
1222 if (isDeadServer(serverName)){
1223 throw new RegionServerStoppedException("hbase:meta says the region "+
1224 regionInfo.getRegionNameAsString()+" is managed by the server " + serverName +
1225 ", but it is dead.");
1226 }
1227
1228
1229 location = new HRegionLocation(regionInfo, serverName,
1230 HRegionInfo.getSeqNumDuringOpen(regionInfoRow));
1231 cacheLocation(tableName, null, location);
1232 return location;
1233 } catch (TableNotFoundException e) {
1234
1235
1236
1237 throw e;
1238 } catch (IOException e) {
1239 if (e instanceof RemoteException) {
1240 e = ((RemoteException)e).unwrapRemoteException();
1241 }
1242 if (tries < numTries - 1) {
1243 if (LOG.isDebugEnabled()) {
1244 LOG.debug("locateRegionInMeta failed; parentTable=" + parentTable +
1245 ", metaLocation=" + ((metaLocation == null)? "null": "{" + metaLocation + "}") +
1246 ", attempt=" + tries + "/" + this.numTries + "; retrying after=" +
1247 ConnectionUtils.getPauseTime(this.pause, tries) + "ms; because: " + e.getMessage());
1248 }
1249 } else {
1250 throw e;
1251 }
1252
1253 if(!(e instanceof RegionOfflineException ||
1254 e instanceof NoServerForRegionException)) {
1255 relocateRegion(parentTable, metaKey);
1256 }
1257 }
1258 try{
1259 Thread.sleep(ConnectionUtils.getPauseTime(this.pause, tries));
1260 } catch (InterruptedException e) {
1261 Thread.currentThread().interrupt();
1262 throw new IOException("Giving up trying to location region in " +
1263 "meta: thread is interrupted.");
1264 }
1265 }
1266 }
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276 HRegionLocation getCachedLocation(final TableName tableName,
1277 final byte [] row) {
1278 ConcurrentSkipListMap<byte[], HRegionLocation> tableLocations =
1279 getTableLocations(tableName);
1280
1281 Entry<byte[], HRegionLocation> e = tableLocations.floorEntry(row);
1282 if (e == null) {
1283 return null;
1284 }
1285 HRegionLocation possibleRegion = e.getValue();
1286
1287
1288
1289
1290
1291
1292 byte[] endKey = possibleRegion.getRegionInfo().getEndKey();
1293 if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
1294 tableName.getRowComparator().compareRows(
1295 endKey, 0, endKey.length, row, 0, row.length) > 0) {
1296 return possibleRegion;
1297 }
1298
1299
1300 return null;
1301 }
1302
1303
1304
1305
1306
1307
1308 void forceDeleteCachedLocation(final TableName tableName, final byte [] row) {
1309 HRegionLocation rl = null;
1310 Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1311
1312
1313 rl = getCachedLocation(tableName, row);
1314 if (rl != null) {
1315 tableLocations.remove(rl.getRegionInfo().getStartKey());
1316 }
1317 if ((rl != null) && LOG.isDebugEnabled()) {
1318 LOG.debug("Removed " + rl.getHostname() + ":" + rl.getPort()
1319 + " as a location of " + rl.getRegionInfo().getRegionNameAsString() +
1320 " for tableName=" + tableName + " from cache");
1321 }
1322 }
1323
1324
1325
1326
1327 @Override
1328 public void clearCaches(final ServerName serverName) {
1329 if (!this.cachedServers.contains(serverName)) {
1330 return;
1331 }
1332
1333 boolean deletedSomething = false;
1334 synchronized (this.cachedServers) {
1335
1336
1337
1338
1339 if (!this.cachedServers.contains(serverName)) {
1340 return;
1341 }
1342 for (Map<byte[], HRegionLocation> tableLocations : cachedRegionLocations.values()) {
1343 for (Entry<byte[], HRegionLocation> e : tableLocations.entrySet()) {
1344 HRegionLocation value = e.getValue();
1345 if (value != null
1346 && serverName.equals(value.getServerName())) {
1347 tableLocations.remove(e.getKey());
1348 deletedSomething = true;
1349 }
1350 }
1351 }
1352 this.cachedServers.remove(serverName);
1353 }
1354 if (deletedSomething && LOG.isDebugEnabled()) {
1355 LOG.debug("Removed all cached region locations that map to " + serverName);
1356 }
1357 }
1358
1359
1360
1361
1362
1363 private ConcurrentSkipListMap<byte[], HRegionLocation> getTableLocations(
1364 final TableName tableName) {
1365
1366 ConcurrentSkipListMap<byte[], HRegionLocation> result;
1367 result = this.cachedRegionLocations.get(tableName);
1368
1369 if (result == null) {
1370 result = new ConcurrentSkipListMap<byte[], HRegionLocation>(Bytes.BYTES_COMPARATOR);
1371 ConcurrentSkipListMap<byte[], HRegionLocation> old =
1372 this.cachedRegionLocations.putIfAbsent(tableName, result);
1373 if (old != null) {
1374 return old;
1375 }
1376 }
1377 return result;
1378 }
1379
1380 @Override
1381 public void clearRegionCache() {
1382 this.cachedRegionLocations.clear();
1383 this.cachedServers.clear();
1384 }
1385
1386 @Override
1387 public void clearRegionCache(final TableName tableName) {
1388 this.cachedRegionLocations.remove(tableName);
1389 }
1390
1391 @Override
1392 public void clearRegionCache(final byte[] tableName) {
1393 clearRegionCache(TableName.valueOf(tableName));
1394 }
1395
1396
1397
1398
1399
1400
1401
1402 private void cacheLocation(final TableName tableName, final HRegionLocation source,
1403 final HRegionLocation location) {
1404 boolean isFromMeta = (source == null);
1405 byte [] startKey = location.getRegionInfo().getStartKey();
1406 ConcurrentMap<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1407 HRegionLocation oldLocation = tableLocations.putIfAbsent(startKey, location);
1408 boolean isNewCacheEntry = (oldLocation == null);
1409 if (isNewCacheEntry) {
1410 cachedServers.add(location.getServerName());
1411 return;
1412 }
1413 boolean updateCache;
1414
1415 if (oldLocation.equals(source)) {
1416 updateCache = true;
1417 } else {
1418 long newLocationSeqNum = location.getSeqNum();
1419
1420
1421 boolean isStaleMetaRecord = isFromMeta && (oldLocation.getSeqNum() > newLocationSeqNum);
1422
1423
1424
1425
1426
1427 boolean isStaleRedirect = !isFromMeta && (oldLocation.getSeqNum() >= newLocationSeqNum);
1428 boolean isStaleUpdate = (isStaleMetaRecord || isStaleRedirect);
1429 updateCache = (!isStaleUpdate);
1430 }
1431 if (updateCache) {
1432 tableLocations.replace(startKey, oldLocation, location);
1433 cachedServers.add(location.getServerName());
1434 }
1435 }
1436
1437
1438 private final ConcurrentHashMap<String, Object> stubs =
1439 new ConcurrentHashMap<String, Object>();
1440
1441 private final ConcurrentHashMap<String, String> connectionLock =
1442 new ConcurrentHashMap<String, String>();
1443
1444
1445
1446
1447 static class MasterServiceState {
1448 HConnection connection;
1449 MasterService.BlockingInterface stub;
1450 int userCount;
1451 long keepAliveUntil = Long.MAX_VALUE;
1452
1453 MasterServiceState (final HConnection connection) {
1454 super();
1455 this.connection = connection;
1456 }
1457
1458 @Override
1459 public String toString() {
1460 return "MasterService";
1461 }
1462
1463 Object getStub() {
1464 return this.stub;
1465 }
1466
1467 void clearStub() {
1468 this.stub = null;
1469 }
1470
1471 boolean isMasterRunning() throws ServiceException {
1472 IsMasterRunningResponse response =
1473 this.stub.isMasterRunning(null, RequestConverter.buildIsMasterRunningRequest());
1474 return response != null? response.getIsMasterRunning(): false;
1475 }
1476 }
1477
1478
1479
1480
1481
1482
1483 abstract class StubMaker {
1484
1485
1486
1487 protected abstract String getServiceName();
1488
1489
1490
1491
1492
1493 protected abstract Object makeStub(final BlockingRpcChannel channel);
1494
1495
1496
1497
1498
1499 protected abstract void isMasterRunning() throws ServiceException;
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509 private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
1510 ZooKeeperKeepAliveConnection zkw;
1511 try {
1512 zkw = getKeepAliveZooKeeperWatcher();
1513 } catch (IOException e) {
1514 throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
1515 }
1516 try {
1517 checkIfBaseNodeAvailable(zkw);
1518 ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
1519 if (sn == null) {
1520 String msg = "ZooKeeper available but no active master location found";
1521 LOG.info(msg);
1522 throw new MasterNotRunningException(msg);
1523 }
1524 if (isDeadServer(sn)) {
1525 throw new MasterNotRunningException(sn + " is dead.");
1526 }
1527
1528 String key = getStubKey(getServiceName(), sn.getHostAndPort());
1529 connectionLock.putIfAbsent(key, key);
1530 Object stub = null;
1531 synchronized (connectionLock.get(key)) {
1532 stub = stubs.get(key);
1533 if (stub == null) {
1534 BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn,
1535 user, rpcTimeout);
1536 stub = makeStub(channel);
1537 isMasterRunning();
1538 stubs.put(key, stub);
1539 }
1540 }
1541 return stub;
1542 } finally {
1543 zkw.close();
1544 }
1545 }
1546
1547
1548
1549
1550
1551
1552 @edu.umd.cs.findbugs.annotations.SuppressWarnings (value="SWL_SLEEP_WITH_LOCK_HELD")
1553 Object makeStub() throws MasterNotRunningException {
1554
1555
1556 synchronized (masterAndZKLock) {
1557 Exception exceptionCaught = null;
1558 Object stub = null;
1559 int tries = 0;
1560 while (!closed && stub == null) {
1561 tries++;
1562 try {
1563 stub = makeStubNoRetries();
1564 } catch (IOException e) {
1565 exceptionCaught = e;
1566 } catch (KeeperException e) {
1567 exceptionCaught = e;
1568 } catch (ServiceException e) {
1569 exceptionCaught = e;
1570 }
1571
1572 if (exceptionCaught != null)
1573
1574 if (tries < numTries) {
1575
1576 long pauseTime = ConnectionUtils.getPauseTime(pause, tries - 1);
1577 LOG.info("getMaster attempt " + tries + " of " + numTries +
1578 " failed; retrying after sleep of " + pauseTime + ", exception=" +
1579 exceptionCaught);
1580
1581 try {
1582 Thread.sleep(pauseTime);
1583 } catch (InterruptedException e) {
1584 Thread.currentThread().interrupt();
1585 throw new RuntimeException(
1586 "Thread was interrupted while trying to connect to master.", e);
1587 }
1588 } else {
1589
1590 LOG.info("getMaster attempt " + tries + " of " + numTries +
1591 " failed; no more retrying.", exceptionCaught);
1592 throw new MasterNotRunningException(exceptionCaught);
1593 }
1594 }
1595
1596 if (stub == null) {
1597
1598 throw new MasterNotRunningException("Connection was closed while trying to get master");
1599 }
1600 return stub;
1601 }
1602 }
1603 }
1604
1605
1606
1607
1608 class MasterServiceStubMaker extends StubMaker {
1609 private MasterService.BlockingInterface stub;
1610 @Override
1611 protected String getServiceName() {
1612 return MasterService.getDescriptor().getName();
1613 }
1614
1615 @Override
1616 @edu.umd.cs.findbugs.annotations.SuppressWarnings("SWL_SLEEP_WITH_LOCK_HELD")
1617 MasterService.BlockingInterface makeStub() throws MasterNotRunningException {
1618 return (MasterService.BlockingInterface)super.makeStub();
1619 }
1620
1621 @Override
1622 protected Object makeStub(BlockingRpcChannel channel) {
1623 this.stub = MasterService.newBlockingStub(channel);
1624 return this.stub;
1625 }
1626
1627 @Override
1628 protected void isMasterRunning() throws ServiceException {
1629 this.stub.isMasterRunning(null, RequestConverter.buildIsMasterRunningRequest());
1630 }
1631 }
1632
1633 @Override
1634 public AdminService.BlockingInterface getAdmin(final ServerName serverName)
1635 throws IOException {
1636 return getAdmin(serverName, false);
1637 }
1638
1639 @Override
1640
1641 public AdminService.BlockingInterface getAdmin(final ServerName serverName,
1642 final boolean master)
1643 throws IOException {
1644 if (isDeadServer(serverName)) {
1645 throw new RegionServerStoppedException(serverName + " is dead.");
1646 }
1647 String key = getStubKey(AdminService.BlockingInterface.class.getName(),
1648 serverName.getHostAndPort());
1649 this.connectionLock.putIfAbsent(key, key);
1650 AdminService.BlockingInterface stub = null;
1651 synchronized (this.connectionLock.get(key)) {
1652 stub = (AdminService.BlockingInterface)this.stubs.get(key);
1653 if (stub == null) {
1654 BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName,
1655 user, this.rpcTimeout);
1656 stub = AdminService.newBlockingStub(channel);
1657 this.stubs.put(key, stub);
1658 }
1659 }
1660 return stub;
1661 }
1662
1663 @Override
1664 public ClientService.BlockingInterface getClient(final ServerName sn)
1665 throws IOException {
1666 if (isDeadServer(sn)) {
1667 throw new RegionServerStoppedException(sn + " is dead.");
1668 }
1669 String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort());
1670 this.connectionLock.putIfAbsent(key, key);
1671 ClientService.BlockingInterface stub = null;
1672 synchronized (this.connectionLock.get(key)) {
1673 stub = (ClientService.BlockingInterface)this.stubs.get(key);
1674 if (stub == null) {
1675 BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(sn,
1676 user, this.rpcTimeout);
1677 stub = ClientService.newBlockingStub(channel);
1678
1679
1680 this.stubs.put(key, stub);
1681 }
1682 }
1683 return stub;
1684 }
1685
1686 static String getStubKey(final String serviceName, final String rsHostnamePort) {
1687 return serviceName + "@" + rsHostnamePort;
1688 }
1689
1690 private ZooKeeperKeepAliveConnection keepAliveZookeeper;
1691 private int keepAliveZookeeperUserCount;
1692 private boolean canCloseZKW = true;
1693
1694
1695 private static final long keepAlive = 5 * 60 * 1000;
1696
1697
1698
1699
1700
1701 ZooKeeperKeepAliveConnection getKeepAliveZooKeeperWatcher()
1702 throws IOException {
1703 synchronized (masterAndZKLock) {
1704 if (keepAliveZookeeper == null) {
1705 if (this.closed) {
1706 throw new IOException(toString() + " closed");
1707 }
1708
1709
1710 keepAliveZookeeper = new ZooKeeperKeepAliveConnection(conf, this.toString(), this);
1711 }
1712 keepAliveZookeeperUserCount++;
1713 keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1714 return keepAliveZookeeper;
1715 }
1716 }
1717
1718 void releaseZooKeeperWatcher(final ZooKeeperWatcher zkw) {
1719 if (zkw == null){
1720 return;
1721 }
1722 synchronized (masterAndZKLock) {
1723 --keepAliveZookeeperUserCount;
1724 if (keepAliveZookeeperUserCount <= 0 ){
1725 keepZooKeeperWatcherAliveUntil = System.currentTimeMillis() + keepAlive;
1726 }
1727 }
1728 }
1729
1730
1731
1732
1733
1734
1735
1736
1737 private static class DelayedClosing extends Chore implements Stoppable {
1738 private HConnectionImplementation hci;
1739 Stoppable stoppable;
1740
1741 private DelayedClosing(
1742 HConnectionImplementation hci, Stoppable stoppable){
1743 super(
1744 "ZooKeeperWatcher and Master delayed closing for connection "+hci,
1745 60*1000,
1746 stoppable);
1747 this.hci = hci;
1748 this.stoppable = stoppable;
1749 }
1750
1751 static DelayedClosing createAndStart(HConnectionImplementation hci){
1752 Stoppable stoppable = new Stoppable() {
1753 private volatile boolean isStopped = false;
1754 @Override public void stop(String why) { isStopped = true;}
1755 @Override public boolean isStopped() {return isStopped;}
1756 };
1757
1758 return new DelayedClosing(hci, stoppable);
1759 }
1760
1761 protected void closeMasterProtocol(MasterServiceState protocolState) {
1762 if (System.currentTimeMillis() > protocolState.keepAliveUntil) {
1763 hci.closeMasterService(protocolState);
1764 protocolState.keepAliveUntil = Long.MAX_VALUE;
1765 }
1766 }
1767
1768 @Override
1769 protected void chore() {
1770 synchronized (hci.masterAndZKLock) {
1771 if (hci.canCloseZKW) {
1772 if (System.currentTimeMillis() >
1773 hci.keepZooKeeperWatcherAliveUntil) {
1774
1775 hci.closeZooKeeperWatcher();
1776 hci.keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1777 }
1778 }
1779 closeMasterProtocol(hci.masterServiceState);
1780 closeMasterProtocol(hci.masterServiceState);
1781 }
1782 }
1783
1784 @Override
1785 public void stop(String why) {
1786 stoppable.stop(why);
1787 }
1788
1789 @Override
1790 public boolean isStopped() {
1791 return stoppable.isStopped();
1792 }
1793 }
1794
1795 private void closeZooKeeperWatcher() {
1796 synchronized (masterAndZKLock) {
1797 if (keepAliveZookeeper != null) {
1798 LOG.info("Closing zookeeper sessionid=0x" +
1799 Long.toHexString(
1800 keepAliveZookeeper.getRecoverableZooKeeper().getSessionId()));
1801 keepAliveZookeeper.internalClose();
1802 keepAliveZookeeper = null;
1803 }
1804 keepAliveZookeeperUserCount = 0;
1805 }
1806 }
1807
1808 final MasterServiceState masterServiceState = new MasterServiceState(this);
1809
1810 @Override
1811 public MasterService.BlockingInterface getMaster() throws MasterNotRunningException {
1812 return getKeepAliveMasterService();
1813 }
1814
1815 private void resetMasterServiceState(final MasterServiceState mss) {
1816 mss.userCount++;
1817 mss.keepAliveUntil = Long.MAX_VALUE;
1818 }
1819
1820 @Override
1821 public MasterKeepAliveConnection getKeepAliveMasterService()
1822 throws MasterNotRunningException {
1823 synchronized (masterAndZKLock) {
1824 if (!isKeepAliveMasterConnectedAndRunning(this.masterServiceState)) {
1825 MasterServiceStubMaker stubMaker = new MasterServiceStubMaker();
1826 this.masterServiceState.stub = stubMaker.makeStub();
1827 }
1828 resetMasterServiceState(this.masterServiceState);
1829 }
1830
1831 final MasterService.BlockingInterface stub = this.masterServiceState.stub;
1832 return new MasterKeepAliveConnection() {
1833 MasterServiceState mss = masterServiceState;
1834 @Override
1835 public AddColumnResponse addColumn(RpcController controller, AddColumnRequest request)
1836 throws ServiceException {
1837 return stub.addColumn(controller, request);
1838 }
1839
1840 @Override
1841 public DeleteColumnResponse deleteColumn(RpcController controller,
1842 DeleteColumnRequest request)
1843 throws ServiceException {
1844 return stub.deleteColumn(controller, request);
1845 }
1846
1847 @Override
1848 public ModifyColumnResponse modifyColumn(RpcController controller,
1849 ModifyColumnRequest request)
1850 throws ServiceException {
1851 return stub.modifyColumn(controller, request);
1852 }
1853
1854 @Override
1855 public MoveRegionResponse moveRegion(RpcController controller,
1856 MoveRegionRequest request) throws ServiceException {
1857 return stub.moveRegion(controller, request);
1858 }
1859
1860 @Override
1861 public DispatchMergingRegionsResponse dispatchMergingRegions(
1862 RpcController controller, DispatchMergingRegionsRequest request)
1863 throws ServiceException {
1864 return stub.dispatchMergingRegions(controller, request);
1865 }
1866
1867 @Override
1868 public AssignRegionResponse assignRegion(RpcController controller,
1869 AssignRegionRequest request) throws ServiceException {
1870 return stub.assignRegion(controller, request);
1871 }
1872
1873 @Override
1874 public UnassignRegionResponse unassignRegion(RpcController controller,
1875 UnassignRegionRequest request) throws ServiceException {
1876 return stub.unassignRegion(controller, request);
1877 }
1878
1879 @Override
1880 public OfflineRegionResponse offlineRegion(RpcController controller,
1881 OfflineRegionRequest request) throws ServiceException {
1882 return stub.offlineRegion(controller, request);
1883 }
1884
1885 @Override
1886 public DeleteTableResponse deleteTable(RpcController controller,
1887 DeleteTableRequest request) throws ServiceException {
1888 return stub.deleteTable(controller, request);
1889 }
1890
1891 @Override
1892 public EnableTableResponse enableTable(RpcController controller,
1893 EnableTableRequest request) throws ServiceException {
1894 return stub.enableTable(controller, request);
1895 }
1896
1897 @Override
1898 public DisableTableResponse disableTable(RpcController controller,
1899 DisableTableRequest request) throws ServiceException {
1900 return stub.disableTable(controller, request);
1901 }
1902
1903 @Override
1904 public ModifyTableResponse modifyTable(RpcController controller,
1905 ModifyTableRequest request) throws ServiceException {
1906 return stub.modifyTable(controller, request);
1907 }
1908
1909 @Override
1910 public CreateTableResponse createTable(RpcController controller,
1911 CreateTableRequest request) throws ServiceException {
1912 return stub.createTable(controller, request);
1913 }
1914
1915 @Override
1916 public ShutdownResponse shutdown(RpcController controller,
1917 ShutdownRequest request) throws ServiceException {
1918 return stub.shutdown(controller, request);
1919 }
1920
1921 @Override
1922 public StopMasterResponse stopMaster(RpcController controller,
1923 StopMasterRequest request) throws ServiceException {
1924 return stub.stopMaster(controller, request);
1925 }
1926
1927 @Override
1928 public BalanceResponse balance(RpcController controller,
1929 BalanceRequest request) throws ServiceException {
1930 return stub.balance(controller, request);
1931 }
1932
1933 @Override
1934 public SetBalancerRunningResponse setBalancerRunning(
1935 RpcController controller, SetBalancerRunningRequest request)
1936 throws ServiceException {
1937 return stub.setBalancerRunning(controller, request);
1938 }
1939
1940 @Override
1941 public RunCatalogScanResponse runCatalogScan(RpcController controller,
1942 RunCatalogScanRequest request) throws ServiceException {
1943 return stub.runCatalogScan(controller, request);
1944 }
1945
1946 @Override
1947 public EnableCatalogJanitorResponse enableCatalogJanitor(
1948 RpcController controller, EnableCatalogJanitorRequest request)
1949 throws ServiceException {
1950 return stub.enableCatalogJanitor(controller, request);
1951 }
1952
1953 @Override
1954 public IsCatalogJanitorEnabledResponse isCatalogJanitorEnabled(
1955 RpcController controller, IsCatalogJanitorEnabledRequest request)
1956 throws ServiceException {
1957 return stub.isCatalogJanitorEnabled(controller, request);
1958 }
1959
1960 @Override
1961 public CoprocessorServiceResponse execMasterService(
1962 RpcController controller, CoprocessorServiceRequest request)
1963 throws ServiceException {
1964 return stub.execMasterService(controller, request);
1965 }
1966
1967 @Override
1968 public SnapshotResponse snapshot(RpcController controller,
1969 SnapshotRequest request) throws ServiceException {
1970 return stub.snapshot(controller, request);
1971 }
1972
1973 @Override
1974 public GetCompletedSnapshotsResponse getCompletedSnapshots(
1975 RpcController controller, GetCompletedSnapshotsRequest request)
1976 throws ServiceException {
1977 return stub.getCompletedSnapshots(controller, request);
1978 }
1979
1980 @Override
1981 public DeleteSnapshotResponse deleteSnapshot(RpcController controller,
1982 DeleteSnapshotRequest request) throws ServiceException {
1983 return stub.deleteSnapshot(controller, request);
1984 }
1985
1986 @Override
1987 public IsSnapshotDoneResponse isSnapshotDone(RpcController controller,
1988 IsSnapshotDoneRequest request) throws ServiceException {
1989 return stub.isSnapshotDone(controller, request);
1990 }
1991
1992 @Override
1993 public RestoreSnapshotResponse restoreSnapshot(
1994 RpcController controller, RestoreSnapshotRequest request)
1995 throws ServiceException {
1996 return stub.restoreSnapshot(controller, request);
1997 }
1998
1999 @Override
2000 public IsRestoreSnapshotDoneResponse isRestoreSnapshotDone(
2001 RpcController controller, IsRestoreSnapshotDoneRequest request)
2002 throws ServiceException {
2003 return stub.isRestoreSnapshotDone(controller, request);
2004 }
2005
2006 @Override
2007 public IsMasterRunningResponse isMasterRunning(
2008 RpcController controller, IsMasterRunningRequest request)
2009 throws ServiceException {
2010 return stub.isMasterRunning(controller, request);
2011 }
2012
2013 @Override
2014 public ModifyNamespaceResponse modifyNamespace(RpcController controller,
2015 ModifyNamespaceRequest request)
2016 throws ServiceException {
2017 return stub.modifyNamespace(controller, request);
2018 }
2019
2020 @Override
2021 public CreateNamespaceResponse createNamespace(RpcController controller, CreateNamespaceRequest request) throws ServiceException {
2022 return stub.createNamespace(controller, request);
2023 }
2024
2025 @Override
2026 public DeleteNamespaceResponse deleteNamespace(RpcController controller, DeleteNamespaceRequest request) throws ServiceException {
2027 return stub.deleteNamespace(controller, request);
2028 }
2029
2030 @Override
2031 public GetNamespaceDescriptorResponse getNamespaceDescriptor(RpcController controller, GetNamespaceDescriptorRequest request) throws ServiceException {
2032 return stub.getNamespaceDescriptor(controller, request);
2033 }
2034
2035 @Override
2036 public ListNamespaceDescriptorsResponse listNamespaceDescriptors(RpcController controller, ListNamespaceDescriptorsRequest request) throws ServiceException {
2037 return stub.listNamespaceDescriptors(controller, request);
2038 }
2039
2040 @Override
2041 public ListTableDescriptorsByNamespaceResponse listTableDescriptorsByNamespace(RpcController controller, ListTableDescriptorsByNamespaceRequest request) throws ServiceException {
2042 return stub.listTableDescriptorsByNamespace(controller, request);
2043 }
2044
2045 @Override
2046 public ListTableNamesByNamespaceResponse listTableNamesByNamespace(RpcController controller,
2047 ListTableNamesByNamespaceRequest request) throws ServiceException {
2048 return stub.listTableNamesByNamespace(controller, request);
2049 }
2050
2051 @Override
2052 public void close() {
2053 release(this.mss);
2054 }
2055
2056 @Override
2057 public GetSchemaAlterStatusResponse getSchemaAlterStatus(
2058 RpcController controller, GetSchemaAlterStatusRequest request)
2059 throws ServiceException {
2060 return stub.getSchemaAlterStatus(controller, request);
2061 }
2062
2063 @Override
2064 public GetTableDescriptorsResponse getTableDescriptors(
2065 RpcController controller, GetTableDescriptorsRequest request)
2066 throws ServiceException {
2067 return stub.getTableDescriptors(controller, request);
2068 }
2069
2070 @Override
2071 public GetTableNamesResponse getTableNames(
2072 RpcController controller, GetTableNamesRequest request)
2073 throws ServiceException {
2074 return stub.getTableNames(controller, request);
2075 }
2076
2077 @Override
2078 public GetClusterStatusResponse getClusterStatus(
2079 RpcController controller, GetClusterStatusRequest request)
2080 throws ServiceException {
2081 return stub.getClusterStatus(controller, request);
2082 }
2083 };
2084 }
2085
2086
2087 private static void release(MasterServiceState mss) {
2088 if (mss != null && mss.connection != null) {
2089 ((HConnectionImplementation)mss.connection).releaseMaster(mss);
2090 }
2091 }
2092
2093 private boolean isKeepAliveMasterConnectedAndRunning(MasterServiceState mss) {
2094 if (mss.getStub() == null){
2095 return false;
2096 }
2097 try {
2098 return mss.isMasterRunning();
2099 } catch (UndeclaredThrowableException e) {
2100
2101
2102 LOG.info("Master connection is not running anymore", e.getUndeclaredThrowable());
2103 return false;
2104 } catch (ServiceException se) {
2105 LOG.warn("Checking master connection", se);
2106 return false;
2107 }
2108 }
2109
2110 void releaseMaster(MasterServiceState mss) {
2111 if (mss.getStub() == null) return;
2112 synchronized (masterAndZKLock) {
2113 --mss.userCount;
2114 if (mss.userCount <= 0) {
2115 mss.keepAliveUntil = System.currentTimeMillis() + keepAlive;
2116 }
2117 }
2118 }
2119
2120 private void closeMasterService(MasterServiceState mss) {
2121 if (mss.getStub() != null) {
2122 LOG.info("Closing master protocol: " + mss);
2123 mss.clearStub();
2124 }
2125 mss.userCount = 0;
2126 }
2127
2128
2129
2130
2131
2132 private void closeMaster() {
2133 synchronized (masterAndZKLock) {
2134 closeMasterService(masterServiceState);
2135 }
2136 }
2137
2138 void updateCachedLocation(HRegionInfo hri, HRegionLocation source,
2139 ServerName serverName, long seqNum) {
2140 HRegionLocation newHrl = new HRegionLocation(hri, serverName, seqNum);
2141 cacheLocation(hri.getTable(), source, newHrl);
2142 }
2143
2144
2145
2146
2147
2148
2149 void deleteCachedLocation(HRegionInfo hri, HRegionLocation source) {
2150 ConcurrentMap<byte[], HRegionLocation> tableLocations = getTableLocations(hri.getTable());
2151 tableLocations.remove(hri.getStartKey(), source);
2152 }
2153
2154 @Override
2155 public void deleteCachedRegionLocation(final HRegionLocation location) {
2156 if (location == null) {
2157 return;
2158 }
2159
2160 HRegionLocation removedLocation;
2161 TableName tableName = location.getRegionInfo().getTable();
2162 Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
2163 removedLocation = tableLocations.remove(location.getRegionInfo().getStartKey());
2164 if (LOG.isDebugEnabled() && removedLocation != null) {
2165 LOG.debug("Removed " +
2166 location.getRegionInfo().getRegionNameAsString() +
2167 " for tableName=" + tableName +
2168 " from cache");
2169 }
2170 }
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180 @Override
2181 public void updateCachedLocations(final TableName tableName, byte[] rowkey,
2182 final Object exception, final HRegionLocation source) {
2183 if (rowkey == null || tableName == null) {
2184 LOG.warn("Coding error, see method javadoc. row=" + (rowkey == null ? "null" : rowkey) +
2185 ", tableName=" + (tableName == null ? "null" : tableName));
2186 return;
2187 }
2188
2189 if (source == null || source.getServerName() == null){
2190
2191 return;
2192 }
2193
2194
2195 final HRegionLocation oldLocation = getCachedLocation(tableName, rowkey);
2196 if (oldLocation == null || !source.getServerName().equals(oldLocation.getServerName())) {
2197
2198
2199 return;
2200 }
2201
2202 HRegionInfo regionInfo = oldLocation.getRegionInfo();
2203 Throwable cause = findException(exception);
2204 if (cause != null) {
2205 if (cause instanceof RegionTooBusyException || cause instanceof RegionOpeningException) {
2206
2207 return;
2208 }
2209
2210 if (cause instanceof RegionMovedException) {
2211 RegionMovedException rme = (RegionMovedException) cause;
2212 if (LOG.isTraceEnabled()) {
2213 LOG.trace("Region " + regionInfo.getRegionNameAsString() + " moved to " +
2214 rme.getHostname() + ":" + rme.getPort() +
2215 " according to " + source.getHostnamePort());
2216 }
2217
2218
2219 updateCachedLocation(
2220 regionInfo, source, rme.getServerName(), rme.getLocationSeqNum());
2221 return;
2222 }
2223 }
2224
2225
2226
2227 deleteCachedLocation(regionInfo, source);
2228 }
2229
2230 @Override
2231 public void updateCachedLocations(final byte[] tableName, byte[] rowkey,
2232 final Object exception, final HRegionLocation source) {
2233 updateCachedLocations(TableName.valueOf(tableName), rowkey, exception, source);
2234 }
2235
2236 @Override
2237 @Deprecated
2238 public void processBatch(List<? extends Row> list,
2239 final TableName tableName,
2240 ExecutorService pool,
2241 Object[] results) throws IOException, InterruptedException {
2242
2243
2244
2245 if (results.length != list.size()) {
2246 throw new IllegalArgumentException(
2247 "argument results must be the same size as argument list");
2248 }
2249 processBatchCallback(list, tableName, pool, results, null);
2250 }
2251
2252 @Override
2253 @Deprecated
2254 public void processBatch(List<? extends Row> list,
2255 final byte[] tableName,
2256 ExecutorService pool,
2257 Object[] results) throws IOException, InterruptedException {
2258 processBatch(list, TableName.valueOf(tableName), pool, results);
2259 }
2260
2261
2262
2263
2264
2265
2266
2267
2268 @Override
2269 @Deprecated
2270 public <R> void processBatchCallback(
2271 List<? extends Row> list,
2272 TableName tableName,
2273 ExecutorService pool,
2274 Object[] results,
2275 Batch.Callback<R> callback)
2276 throws IOException, InterruptedException {
2277
2278
2279
2280 ObjectResultFiller<R> cb = new ObjectResultFiller<R>(results, callback);
2281 AsyncProcess<?> asyncProcess = createAsyncProcess(tableName, pool, cb, conf);
2282
2283
2284 asyncProcess.submitAll(list);
2285 asyncProcess.waitUntilDone();
2286
2287 if (asyncProcess.hasError()) {
2288 throw asyncProcess.getErrors();
2289 }
2290 }
2291
2292 @Override
2293 @Deprecated
2294 public <R> void processBatchCallback(
2295 List<? extends Row> list,
2296 byte[] tableName,
2297 ExecutorService pool,
2298 Object[] results,
2299 Batch.Callback<R> callback)
2300 throws IOException, InterruptedException {
2301 processBatchCallback(list, TableName.valueOf(tableName), pool, results, callback);
2302 }
2303
2304
2305 protected <R> AsyncProcess createAsyncProcess(TableName tableName, ExecutorService pool,
2306 AsyncProcess.AsyncProcessCallback<R> callback, Configuration conf) {
2307 return new AsyncProcess<R>(this, tableName, pool, callback, conf,
2308 RpcRetryingCallerFactory.instantiate(conf));
2309 }
2310
2311
2312
2313
2314
2315 private static class ObjectResultFiller<Res>
2316 implements AsyncProcess.AsyncProcessCallback<Res> {
2317
2318 private final Object[] results;
2319 private Batch.Callback<Res> callback;
2320
2321 ObjectResultFiller(Object[] results, Batch.Callback<Res> callback) {
2322 this.results = results;
2323 this.callback = callback;
2324 }
2325
2326 @Override
2327 public void success(int pos, byte[] region, Row row, Res result) {
2328 assert pos < results.length;
2329 results[pos] = result;
2330 if (callback != null) {
2331 callback.update(region, row.getRow(), result);
2332 }
2333 }
2334
2335 @Override
2336 public boolean failure(int pos, byte[] region, Row row, Throwable t) {
2337 assert pos < results.length;
2338 results[pos] = t;
2339
2340 return true;
2341 }
2342
2343 @Override
2344 public boolean retriableFailure(int originalIndex, Row row, byte[] region,
2345 Throwable exception) {
2346 return true;
2347 }
2348 }
2349
2350
2351
2352
2353
2354
2355 int getNumberOfCachedRegionLocations(final TableName tableName) {
2356 Map<byte[], HRegionLocation> tableLocs = this.cachedRegionLocations.get(tableName);
2357 if (tableLocs == null) {
2358 return 0;
2359 }
2360 return tableLocs.values().size();
2361 }
2362
2363
2364
2365
2366
2367
2368
2369
2370 boolean isRegionCached(TableName tableName, final byte[] row) {
2371 HRegionLocation location = getCachedLocation(tableName, row);
2372 return location != null;
2373 }
2374
2375 @Override
2376 public void setRegionCachePrefetch(final TableName tableName,
2377 final boolean enable) {
2378 if (!enable) {
2379 regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName.getName()));
2380 }
2381 else {
2382 regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName.getName()));
2383 }
2384 }
2385
2386 @Override
2387 public void setRegionCachePrefetch(final byte[] tableName,
2388 final boolean enable) {
2389 setRegionCachePrefetch(TableName.valueOf(tableName), enable);
2390 }
2391
2392 @Override
2393 public boolean getRegionCachePrefetch(TableName tableName) {
2394 return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName.getName()));
2395 }
2396
2397 @Override
2398 public boolean getRegionCachePrefetch(byte[] tableName) {
2399 return getRegionCachePrefetch(TableName.valueOf(tableName));
2400 }
2401
2402 @Override
2403 public void abort(final String msg, Throwable t) {
2404 if (t instanceof KeeperException.SessionExpiredException
2405 && keepAliveZookeeper != null) {
2406 synchronized (masterAndZKLock) {
2407 if (keepAliveZookeeper != null) {
2408 LOG.warn("This client just lost it's session with ZooKeeper," +
2409 " closing it." +
2410 " It will be recreated next time someone needs it", t);
2411 closeZooKeeperWatcher();
2412 }
2413 }
2414 } else {
2415 if (t != null) {
2416 LOG.fatal(msg, t);
2417 } else {
2418 LOG.fatal(msg);
2419 }
2420 this.aborted = true;
2421 close();
2422 this.closed = true;
2423 }
2424 }
2425
2426 @Override
2427 public boolean isClosed() {
2428 return this.closed;
2429 }
2430
2431 @Override
2432 public boolean isAborted(){
2433 return this.aborted;
2434 }
2435
2436 @Override
2437 public int getCurrentNrHRS() throws IOException {
2438 return this.registry.getCurrentNrHRS();
2439 }
2440
2441
2442
2443
2444 void incCount() {
2445 ++refCount;
2446 }
2447
2448
2449
2450
2451 void decCount() {
2452 if (refCount > 0) {
2453 --refCount;
2454 }
2455 }
2456
2457
2458
2459
2460
2461
2462 boolean isZeroReference() {
2463 return refCount == 0;
2464 }
2465
2466 void internalClose() {
2467 if (this.closed) {
2468 return;
2469 }
2470 delayedClosing.stop("Closing connection");
2471 closeMaster();
2472 shutdownBatchPool();
2473 this.closed = true;
2474 closeZooKeeperWatcher();
2475 this.stubs.clear();
2476 if (clusterStatusListener != null) {
2477 clusterStatusListener.close();
2478 }
2479 if (rpcClient != null) {
2480 rpcClient.stop();
2481 }
2482 }
2483
2484 @Override
2485 public void close() {
2486 if (managed) {
2487 if (aborted) {
2488 HConnectionManager.deleteStaleConnection(this);
2489 } else {
2490 HConnectionManager.deleteConnection(this, false);
2491 }
2492 } else {
2493 internalClose();
2494 }
2495 }
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508 @Override
2509 protected void finalize() throws Throwable {
2510 super.finalize();
2511
2512 refCount = 1;
2513 close();
2514 }
2515
2516 @Override
2517 public HTableDescriptor[] listTables() throws IOException {
2518 MasterKeepAliveConnection master = getKeepAliveMasterService();
2519 try {
2520 GetTableDescriptorsRequest req =
2521 RequestConverter.buildGetTableDescriptorsRequest((List<TableName>)null);
2522 return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2523 } catch (ServiceException se) {
2524 throw ProtobufUtil.getRemoteException(se);
2525 } finally {
2526 master.close();
2527 }
2528 }
2529
2530 @Override
2531 public String[] getTableNames() throws IOException {
2532 TableName[] tableNames = listTableNames();
2533 String result[] = new String[tableNames.length];
2534 for (int i = 0; i < tableNames.length; i++) {
2535 result[i] = tableNames[i].getNameAsString();
2536 }
2537 return result;
2538 }
2539
2540 @Override
2541 public TableName[] listTableNames() throws IOException {
2542 MasterKeepAliveConnection master = getKeepAliveMasterService();
2543 try {
2544 return ProtobufUtil.getTableNameArray(master.getTableNames(null,
2545 GetTableNamesRequest.newBuilder().build())
2546 .getTableNamesList());
2547 } catch (ServiceException se) {
2548 throw ProtobufUtil.getRemoteException(se);
2549 } finally {
2550 master.close();
2551 }
2552 }
2553
2554 @Override
2555 public HTableDescriptor[] getHTableDescriptorsByTableName(
2556 List<TableName> tableNames) throws IOException {
2557 if (tableNames == null || tableNames.isEmpty()) return new HTableDescriptor[0];
2558 MasterKeepAliveConnection master = getKeepAliveMasterService();
2559 try {
2560 GetTableDescriptorsRequest req =
2561 RequestConverter.buildGetTableDescriptorsRequest(tableNames);
2562 return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2563 } catch (ServiceException se) {
2564 throw ProtobufUtil.getRemoteException(se);
2565 } finally {
2566 master.close();
2567 }
2568 }
2569
2570 @Override
2571 public HTableDescriptor[] getHTableDescriptors(
2572 List<String> names) throws IOException {
2573 List<TableName> tableNames = new ArrayList(names.size());
2574 for(String name : names) {
2575 tableNames.add(TableName.valueOf(name));
2576 }
2577
2578 return getHTableDescriptorsByTableName(tableNames);
2579 }
2580
2581
2582
2583
2584
2585
2586
2587
2588 @Override
2589 public HTableDescriptor getHTableDescriptor(final TableName tableName)
2590 throws IOException {
2591 if (tableName == null) return null;
2592 if (tableName.equals(TableName.META_TABLE_NAME)) {
2593 return HTableDescriptor.META_TABLEDESC;
2594 }
2595 MasterKeepAliveConnection master = getKeepAliveMasterService();
2596 GetTableDescriptorsResponse htds;
2597 try {
2598 GetTableDescriptorsRequest req =
2599 RequestConverter.buildGetTableDescriptorsRequest(tableName);
2600 htds = master.getTableDescriptors(null, req);
2601 } catch (ServiceException se) {
2602 throw ProtobufUtil.getRemoteException(se);
2603 } finally {
2604 master.close();
2605 }
2606 if (!htds.getTableSchemaList().isEmpty()) {
2607 return HTableDescriptor.convert(htds.getTableSchemaList().get(0));
2608 }
2609 throw new TableNotFoundException(tableName.getNameAsString());
2610 }
2611
2612 @Override
2613 public HTableDescriptor getHTableDescriptor(final byte[] tableName)
2614 throws IOException {
2615 return getHTableDescriptor(TableName.valueOf(tableName));
2616 }
2617 }
2618
2619
2620
2621
2622 static class ServerErrorTracker {
2623
2624 private final ConcurrentMap<HRegionLocation, ServerErrors> errorsByServer =
2625 new ConcurrentHashMap<HRegionLocation, ServerErrors>();
2626 private final long canRetryUntil;
2627 private final int maxRetries;
2628 private final String startTrackingTime;
2629
2630 public ServerErrorTracker(long timeout, int maxRetries) {
2631 this.maxRetries = maxRetries;
2632 this.canRetryUntil = EnvironmentEdgeManager.currentTimeMillis() + timeout;
2633 this.startTrackingTime = new Date().toString();
2634 }
2635
2636
2637
2638
2639 boolean canRetryMore(int numRetry) {
2640
2641 return numRetry < maxRetries || (maxRetries > 1 &&
2642 EnvironmentEdgeManager.currentTimeMillis() < this.canRetryUntil);
2643 }
2644
2645
2646
2647
2648
2649
2650
2651
2652 long calculateBackoffTime(HRegionLocation server, long basePause) {
2653 long result;
2654 ServerErrors errorStats = errorsByServer.get(server);
2655 if (errorStats != null) {
2656 result = ConnectionUtils.getPauseTime(basePause, errorStats.retries.get());
2657 } else {
2658 result = 0;
2659 }
2660 return result;
2661 }
2662
2663
2664
2665
2666
2667
2668 void reportServerError(HRegionLocation server) {
2669 ServerErrors errors = errorsByServer.get(server);
2670 if (errors != null) {
2671 errors.addError();
2672 } else {
2673 errors = errorsByServer.putIfAbsent(server, new ServerErrors());
2674 if (errors != null){
2675 errors.addError();
2676 }
2677 }
2678 }
2679
2680 String getStartTrackingTime() {
2681 return startTrackingTime;
2682 }
2683
2684
2685
2686
2687 private static class ServerErrors {
2688 public final AtomicInteger retries = new AtomicInteger(0);
2689
2690 public void addError() {
2691 retries.incrementAndGet();
2692 }
2693 }
2694 }
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704 public static Throwable findException(Object exception) {
2705 if (exception == null || !(exception instanceof Throwable)) {
2706 return null;
2707 }
2708 Throwable cur = (Throwable) exception;
2709 while (cur != null) {
2710 if (cur instanceof RegionMovedException || cur instanceof RegionOpeningException
2711 || cur instanceof RegionTooBusyException) {
2712 return cur;
2713 }
2714 if (cur instanceof RemoteException) {
2715 RemoteException re = (RemoteException) cur;
2716 cur = re.unwrapRemoteException(
2717 RegionOpeningException.class, RegionMovedException.class,
2718 RegionTooBusyException.class);
2719 if (cur == null) {
2720 cur = re.unwrapRemoteException();
2721 }
2722
2723
2724
2725 if (cur == re) {
2726 return null;
2727 }
2728 } else {
2729 cur = cur.getCause();
2730 }
2731 }
2732
2733 return null;
2734 }
2735
2736
2737
2738
2739
2740
2741
2742
2743 public static void setServerSideHConnectionRetries(final Configuration c, final String sn,
2744 final Log log) {
2745 int hcRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2746 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2747
2748
2749 int serversideMultiplier = c.getInt("hbase.client.serverside.retries.multiplier", 10);
2750 int retries = hcRetries * serversideMultiplier;
2751 c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, retries);
2752 log.debug(sn + " HConnection server-to-server retries=" + retries);
2753 }
2754 }