View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.lang.reflect.Constructor;
24  import java.lang.reflect.UndeclaredThrowableException;
25  import java.net.SocketException;
26  import java.util.ArrayList;
27  import java.util.Date;
28  import java.util.HashSet;
29  import java.util.LinkedHashMap;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.Map.Entry;
33  import java.util.NavigableMap;
34  import java.util.Set;
35  import java.util.concurrent.ConcurrentHashMap;
36  import java.util.concurrent.ConcurrentMap;
37  import java.util.concurrent.ConcurrentSkipListMap;
38  import java.util.concurrent.ConcurrentSkipListSet;
39  import java.util.concurrent.CopyOnWriteArraySet;
40  import java.util.concurrent.ExecutorService;
41  import java.util.concurrent.LinkedBlockingQueue;
42  import java.util.concurrent.ThreadPoolExecutor;
43  import java.util.concurrent.TimeUnit;
44  import java.util.concurrent.atomic.AtomicBoolean;
45  import java.util.concurrent.atomic.AtomicInteger;
46  
47  import org.apache.commons.logging.Log;
48  import org.apache.commons.logging.LogFactory;
49  import org.apache.hadoop.classification.InterfaceAudience;
50  import org.apache.hadoop.classification.InterfaceStability;
51  import org.apache.hadoop.conf.Configuration;
52  import org.apache.hadoop.hbase.Chore;
53  import org.apache.hadoop.hbase.HBaseConfiguration;
54  import org.apache.hadoop.hbase.HConstants;
55  import org.apache.hadoop.hbase.HRegionInfo;
56  import org.apache.hadoop.hbase.HRegionLocation;
57  import org.apache.hadoop.hbase.HTableDescriptor;
58  import org.apache.hadoop.hbase.MasterNotRunningException;
59  import org.apache.hadoop.hbase.RegionTooBusyException;
60  import org.apache.hadoop.hbase.ServerName;
61  import org.apache.hadoop.hbase.Stoppable;
62  import org.apache.hadoop.hbase.TableName;
63  import org.apache.hadoop.hbase.TableNotEnabledException;
64  import org.apache.hadoop.hbase.TableNotFoundException;
65  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
66  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
67  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
68  import org.apache.hadoop.hbase.client.coprocessor.Batch;
69  import org.apache.hadoop.hbase.exceptions.RegionMovedException;
70  import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
71  import org.apache.hadoop.hbase.ipc.RpcClient;
72  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73  import org.apache.hadoop.hbase.protobuf.RequestConverter;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
75  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
76  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
77  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
78  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnResponse;
79  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionResponse;
80  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableResponse;
82  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnResponse;
83  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableResponse;
85  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse;
86  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse;
87  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
88  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse;
89  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsResponse;
90  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
91  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
92  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest;
93  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse;
94  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse;
96  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
97  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
98  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceResponse;
99  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnResponse;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableResponse;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionResponse;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionResponse;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterResponse;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionResponse;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.*;
115 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
116 import org.apache.hadoop.hbase.security.User;
117 import org.apache.hadoop.hbase.security.UserProvider;
118 import org.apache.hadoop.hbase.util.Bytes;
119 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
120 import org.apache.hadoop.hbase.util.Threads;
121 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
122 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
123 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
124 import org.apache.hadoop.ipc.RemoteException;
125 import org.apache.zookeeper.KeeperException;
126 
127 import com.google.protobuf.BlockingRpcChannel;
128 import com.google.protobuf.RpcController;
129 import com.google.protobuf.ServiceException;
130 
131 /**
132  * A non-instantiable class that manages creation of {@link HConnection}s.
133  * <p>The simplest way to use this class is by using {@link #createConnection(Configuration)}.
134  * This creates a new {@link HConnection} to the cluster that is managed by the caller.
135  * From this {@link HConnection} {@link HTableInterface} implementations are retrieved
136  * with {@link HConnection#getTable(byte[])}. Example:
137  * <pre>
138  * {@code
139  * HConnection connection = HConnectionManager.createConnection(config);
140  * HTableInterface table = connection.getTable("table1");
141  * try {
142  *   // Use the table as needed, for a single operation and a single thread
143  * } finally {
144  *   table.close();
145  *   connection.close();
146  * }} </pre>
147  * <p>The following logic and API will be removed in the future:
148  * <p>This class has a static Map of {@link HConnection} instances keyed by
149  * {@link Configuration}; all invocations of {@link #getConnection(Configuration)}
150  * that pass the same {@link Configuration} instance will be returned the same
151  * {@link  HConnection} instance (Adding properties to a Configuration
152  * instance does not change its object identity; for more on how this is done see
153  * {@link HConnectionKey}).  Sharing {@link HConnection}
154  * instances is usually what you want; all clients of the {@link HConnection}
155  * instances share the HConnections' cache of Region locations rather than each
156  * having to discover for itself the location of meta, etc.  It makes
157  * sense for the likes of the pool of HTables class {@link HTablePool}, for
158  * instance (If concerned that a single {@link HConnection} is insufficient
159  * for sharing amongst clients in say an heavily-multithreaded environment,
160  * in practise its not proven to be an issue.  Besides, {@link HConnection} is
161  * implemented atop Hadoop RPC and as of this writing, Hadoop RPC does a
162  * connection per cluster-member, exclusively).
163  *
164  * <p>But sharing connections makes clean up of {@link HConnection} instances a little awkward.
165  * Currently, clients cleanup by calling {@link #deleteConnection(Configuration)}. This will
166  * shutdown the zookeeper connection the HConnection was using and clean up all
167  * HConnection resources as well as stopping proxies to servers out on the
168  * cluster. Not running the cleanup will not end the world; it'll
169  * just stall the closeup some and spew some zookeeper connection failed
170  * messages into the log.  Running the cleanup on a {@link HConnection} that is
171  * subsequently used by another will cause breakage so be careful running
172  * cleanup.
173  * <p>To create a {@link HConnection} that is not shared by others, you can
174  * create a new {@link Configuration} instance, pass this new instance to
175  * {@link #getConnection(Configuration)}, and then when done, close it up by
176  * doing something like the following:
177  * <pre>
178  * {@code
179  * Configuration newConfig = new Configuration(originalConf);
180  * HConnection connection = HConnectionManager.getConnection(newConfig);
181  * // Use the connection to your hearts' delight and then when done...
182  * HConnectionManager.deleteConnection(newConfig, true);
183  * }
184  * </pre>
185  * <p>Cleanup used to be done inside in a shutdown hook.  On startup we'd
186  * register a shutdown hook that called {@link #deleteAllConnections()}
187  * on its way out but the order in which shutdown hooks run is not defined so
188  * were problematic for clients of HConnection that wanted to register their
189  * own shutdown hooks so we removed ours though this shifts the onus for
190  * cleanup to the client.
191  */
192 @SuppressWarnings("serial")
193 @InterfaceAudience.Public
194 @InterfaceStability.Evolving
195 public class HConnectionManager {
196   static final Log LOG = LogFactory.getLog(HConnectionManager.class);
197 
198   public static final String RETRIES_BY_SERVER_KEY = "hbase.client.retries.by.server";
199 
200   // An LRU Map of HConnectionKey -> HConnection (TableServer).  All
201   // access must be synchronized.  This map is not private because tests
202   // need to be able to tinker with it.
203   static final Map<HConnectionKey, HConnectionImplementation> CONNECTION_INSTANCES;
204 
205   public static final int MAX_CACHED_CONNECTION_INSTANCES;
206 
207   static {
208     // We set instances to one more than the value specified for {@link
209     // HConstants#ZOOKEEPER_MAX_CLIENT_CNXNS}. By default, the zk default max
210     // connections to the ensemble from the one client is 30, so in that case we
211     // should run into zk issues before the LRU hit this value of 31.
212     MAX_CACHED_CONNECTION_INSTANCES = HBaseConfiguration.create().getInt(
213       HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1;
214     CONNECTION_INSTANCES = new LinkedHashMap<HConnectionKey, HConnectionImplementation>(
215         (int) (MAX_CACHED_CONNECTION_INSTANCES / 0.75F) + 1, 0.75F, true) {
216       @Override
217       protected boolean removeEldestEntry(
218           Map.Entry<HConnectionKey, HConnectionImplementation> eldest) {
219          return size() > MAX_CACHED_CONNECTION_INSTANCES;
220        }
221     };
222   }
223 
224   /*
225    * Non-instantiable.
226    */
227   private HConnectionManager() {
228     super();
229   }
230 
231   /**
232    * Get the connection that goes with the passed <code>conf</code> configuration instance.
233    * If no current connection exists, method creates a new connection and keys it using
234    * connection-specific properties from the passed {@link Configuration}; see
235    * {@link HConnectionKey}.
236    * @param conf configuration
237    * @return HConnection object for <code>conf</code>
238    * @throws ZooKeeperConnectionException
239    */
240   @Deprecated
241   public static HConnection getConnection(final Configuration conf)
242   throws IOException {
243     HConnectionKey connectionKey = new HConnectionKey(conf);
244     synchronized (CONNECTION_INSTANCES) {
245       HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);
246       if (connection == null) {
247         connection = (HConnectionImplementation)createConnection(conf, true);
248         CONNECTION_INSTANCES.put(connectionKey, connection);
249       } else if (connection.isClosed()) {
250         HConnectionManager.deleteConnection(connectionKey, true);
251         connection = (HConnectionImplementation)createConnection(conf, true);
252         CONNECTION_INSTANCES.put(connectionKey, connection);
253       }
254       connection.incCount();
255       return connection;
256     }
257   }
258 
259   /**
260    * Create a new HConnection instance using the passed <code>conf</code> instance.
261    * <p>Note: This bypasses the usual HConnection life cycle management done by
262    * {@link #getConnection(Configuration)}. The caller is responsible for
263    * calling {@link HConnection#close()} on the returned connection instance.
264    *
265    * This is the recommended way to create HConnections.
266    * {@code
267    * HConnection connection = HConnectionManager.createConnection(conf);
268    * HTableInterface table = connection.getTable("mytable");
269    * table.get(...);
270    * ...
271    * table.close();
272    * connection.close();
273    * }
274    *
275    * @param conf configuration
276    * @return HConnection object for <code>conf</code>
277    * @throws ZooKeeperConnectionException
278    */
279   public static HConnection createConnection(Configuration conf)
280   throws IOException {
281     UserProvider provider = UserProvider.instantiate(conf);
282     return createConnection(conf, false, null, provider.getCurrent());
283   }
284 
285   /**
286    * Create a new HConnection instance using the passed <code>conf</code> instance.
287    * <p>Note: This bypasses the usual HConnection life cycle management done by
288    * {@link #getConnection(Configuration)}. The caller is responsible for
289    * calling {@link HConnection#close()} on the returned connection instance.
290    * This is the recommended way to create HConnections.
291    * {@code
292    * ExecutorService pool = ...;
293    * HConnection connection = HConnectionManager.createConnection(conf, pool);
294    * HTableInterface table = connection.getTable("mytable");
295    * table.get(...);
296    * ...
297    * table.close();
298    * connection.close();
299    * }
300    * @param conf configuration
301    * @param pool the thread pool to use for batch operation in HTables used via this HConnection
302    * @return HConnection object for <code>conf</code>
303    * @throws ZooKeeperConnectionException
304    */
305   public static HConnection createConnection(Configuration conf, ExecutorService pool)
306   throws IOException {
307     UserProvider provider = UserProvider.instantiate(conf);
308     return createConnection(conf, false, pool, provider.getCurrent());
309   }
310 
311   /**
312    * Create a new HConnection instance using the passed <code>conf</code> instance.
313    * <p>Note: This bypasses the usual HConnection life cycle management done by
314    * {@link #getConnection(Configuration)}. The caller is responsible for
315    * calling {@link HConnection#close()} on the returned connection instance.
316    * This is the recommended way to create HConnections.
317    * {@code
318    * ExecutorService pool = ...;
319    * HConnection connection = HConnectionManager.createConnection(conf, pool);
320    * HTableInterface table = connection.getTable("mytable");
321    * table.get(...);
322    * ...
323    * table.close();
324    * connection.close();
325    * }
326    * @param conf configuration
327    * @param user the user the connection is for
328    * @return HConnection object for <code>conf</code>
329    * @throws ZooKeeperConnectionException
330    */
331   public static HConnection createConnection(Configuration conf, User user)
332   throws IOException {
333     return createConnection(conf, false, null, user);
334   }
335 
336   /**
337    * Create a new HConnection instance using the passed <code>conf</code> instance.
338    * <p>Note: This bypasses the usual HConnection life cycle management done by
339    * {@link #getConnection(Configuration)}. The caller is responsible for
340    * calling {@link HConnection#close()} on the returned connection instance.
341    * This is the recommended way to create HConnections.
342    * {@code
343    * ExecutorService pool = ...;
344    * HConnection connection = HConnectionManager.createConnection(conf, pool);
345    * HTableInterface table = connection.getTable("mytable");
346    * table.get(...);
347    * ...
348    * table.close();
349    * connection.close();
350    * }
351    * @param conf configuration
352    * @param pool the thread pool to use for batch operation in HTables used via this HConnection
353    * @param user the user the connection is for
354    * @return HConnection object for <code>conf</code>
355    * @throws ZooKeeperConnectionException
356    */
357   public static HConnection createConnection(Configuration conf, ExecutorService pool, User user)
358   throws IOException {
359     return createConnection(conf, false, pool, user);
360   }
361 
362   @Deprecated
363   static HConnection createConnection(final Configuration conf, final boolean managed)
364       throws IOException {
365     UserProvider provider = UserProvider.instantiate(conf);
366     return createConnection(conf, managed, null, provider.getCurrent());
367   }
368 
369   @Deprecated
370   static HConnection createConnection(final Configuration conf, final boolean managed,
371       final ExecutorService pool, final User user)
372   throws IOException {
373     String className = conf.get("hbase.client.connection.impl",
374       HConnectionManager.HConnectionImplementation.class.getName());
375     Class<?> clazz = null;
376     try {
377       clazz = Class.forName(className);
378     } catch (ClassNotFoundException e) {
379       throw new IOException(e);
380     }
381     try {
382       // Default HCM#HCI is not accessible; make it so before invoking.
383       Constructor<?> constructor =
384         clazz.getDeclaredConstructor(Configuration.class,
385           boolean.class, ExecutorService.class, User.class);
386       constructor.setAccessible(true);
387       return (HConnection) constructor.newInstance(conf, managed, pool, user);
388     } catch (Exception e) {
389       throw new IOException(e);
390     }
391   }
392 
393   /**
394    * Delete connection information for the instance specified by passed configuration.
395    * If there are no more references to the designated connection connection, this method will
396    * then close connection to the zookeeper ensemble and let go of all associated resources.
397    *
398    * @param conf configuration whose identity is used to find {@link HConnection} instance.
399    * @deprecated
400    */
401   public static void deleteConnection(Configuration conf) {
402     deleteConnection(new HConnectionKey(conf), false);
403   }
404 
405   /**
406    * Cleanup a known stale connection.
407    * This will then close connection to the zookeeper ensemble and let go of all resources.
408    *
409    * @param connection
410    * @deprecated
411    */
412   public static void deleteStaleConnection(HConnection connection) {
413     deleteConnection(connection, true);
414   }
415 
416   /**
417    * Delete information for all connections. Close or not the connection, depending on the
418    *  staleConnection boolean and the ref count. By default, you should use it with
419    *  staleConnection to true.
420    * @deprecated
421    */
422   public static void deleteAllConnections(boolean staleConnection) {
423     synchronized (CONNECTION_INSTANCES) {
424       Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
425       connectionKeys.addAll(CONNECTION_INSTANCES.keySet());
426       for (HConnectionKey connectionKey : connectionKeys) {
427         deleteConnection(connectionKey, staleConnection);
428       }
429       CONNECTION_INSTANCES.clear();
430     }
431   }
432 
433   /**
434    * Delete information for all connections..
435    * @deprecated kept for backward compatibility, but the behavior is broken. HBASE-8983
436    */
437   @Deprecated
438   public static void deleteAllConnections() {
439     deleteAllConnections(false);
440   }
441 
442 
443   @Deprecated
444   private static void deleteConnection(HConnection connection, boolean staleConnection) {
445     synchronized (CONNECTION_INSTANCES) {
446       for (Entry<HConnectionKey, HConnectionImplementation> e: CONNECTION_INSTANCES.entrySet()) {
447         if (e.getValue() == connection) {
448           deleteConnection(e.getKey(), staleConnection);
449           break;
450         }
451       }
452     }
453   }
454 
455   @Deprecated
456   private static void deleteConnection(HConnectionKey connectionKey, boolean staleConnection) {
457     synchronized (CONNECTION_INSTANCES) {
458       HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);
459       if (connection != null) {
460         connection.decCount();
461         if (connection.isZeroReference() || staleConnection) {
462           CONNECTION_INSTANCES.remove(connectionKey);
463           connection.internalClose();
464         }
465       } else {
466         LOG.error("Connection not found in the list, can't delete it "+
467           "(connection key=" + connectionKey + "). May be the key was modified?", new Exception());
468       }
469     }
470   }
471 
472   /**
473    * It is provided for unit test cases which verify the behavior of region
474    * location cache prefetch.
475    * @return Number of cached regions for the table.
476    * @throws ZooKeeperConnectionException
477    */
478   static int getCachedRegionCount(Configuration conf, final TableName tableName)
479   throws IOException {
480     return execute(new HConnectable<Integer>(conf) {
481       @Override
482       public Integer connect(HConnection connection) {
483         return ((HConnectionImplementation)connection).getNumberOfCachedRegionLocations(tableName);
484       }
485     });
486   }
487 
488   /**
489    * It's provided for unit test cases which verify the behavior of region
490    * location cache prefetch.
491    * @return true if the region where the table and row reside is cached.
492    * @throws ZooKeeperConnectionException
493    */
494   static boolean isRegionCached(Configuration conf,
495                                 final TableName tableName,
496                                 final byte[] row)
497   throws IOException {
498     return execute(new HConnectable<Boolean>(conf) {
499       @Override
500       public Boolean connect(HConnection connection) {
501         return ((HConnectionImplementation) connection).isRegionCached(tableName, row);
502       }
503     });
504   }
505 
506   /**
507    * This convenience method invokes the given {@link HConnectable#connect}
508    * implementation using a {@link HConnection} instance that lasts just for the
509    * duration of the invocation.
510    *
511    * @param <T> the return type of the connect method
512    * @param connectable the {@link HConnectable} instance
513    * @return the value returned by the connect method
514    * @throws IOException
515    */
516   @InterfaceAudience.Private
517   public static <T> T execute(HConnectable<T> connectable) throws IOException {
518     if (connectable == null || connectable.conf == null) {
519       return null;
520     }
521     Configuration conf = connectable.conf;
522     HConnection connection = HConnectionManager.getConnection(conf);
523     boolean connectSucceeded = false;
524     try {
525       T returnValue = connectable.connect(connection);
526       connectSucceeded = true;
527       return returnValue;
528     } finally {
529       try {
530         connection.close();
531       } catch (Exception e) {
532         if (connectSucceeded) {
533           throw new IOException("The connection to " + connection
534               + " could not be deleted.", e);
535         }
536       }
537     }
538   }
539 
540   /** Encapsulates connection to zookeeper and regionservers.*/
541   @edu.umd.cs.findbugs.annotations.SuppressWarnings(
542       value="AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION",
543       justification="Access to the conncurrent hash map is under a lock so should be fine.")
544   static class HConnectionImplementation implements HConnection, Closeable {
545     static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);
546     private final long pause;
547     private final int numTries;
548     final int rpcTimeout;
549     private final int prefetchRegionLimit;
550 
551     private volatile boolean closed;
552     private volatile boolean aborted;
553 
554     // package protected for the tests
555     ClusterStatusListener clusterStatusListener;
556 
557     private final Object userRegionLock = new Object();
558 
559     // We have a single lock for master & zk to prevent deadlocks. Having
560     //  one lock for ZK and one lock for master is not possible:
561     //  When creating a connection to master, we need a connection to ZK to get
562     //  its address. But another thread could have taken the ZK lock, and could
563     //  be waiting for the master lock => deadlock.
564     private final Object masterAndZKLock = new Object();
565 
566     private long keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
567     private final DelayedClosing delayedClosing =
568       DelayedClosing.createAndStart(this);
569 
570     // thread executor shared by all HTableInterface instances created
571     // by this connection
572     private volatile ExecutorService batchPool = null;
573     private volatile boolean cleanupPool = false;
574 
575     private final Configuration conf;
576 
577     // Client rpc instance.
578     private RpcClient rpcClient;
579 
580     /**
581       * Map of table to table {@link HRegionLocation}s.
582       */
583     private final ConcurrentMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>
584         cachedRegionLocations =
585       new ConcurrentHashMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>();
586 
587     // The presence of a server in the map implies it's likely that there is an
588     // entry in cachedRegionLocations that map to this server; but the absence
589     // of a server in this map guarentees that there is no entry in cache that
590     // maps to the absent server.
591     // The access to this attribute must be protected by a lock on cachedRegionLocations
592     private final Set<ServerName> cachedServers = new ConcurrentSkipListSet<ServerName>();
593 
594     // region cache prefetch is enabled by default. this set contains all
595     // tables whose region cache prefetch are disabled.
596     private final Set<Integer> regionCachePrefetchDisabledTables =
597       new CopyOnWriteArraySet<Integer>();
598 
599     private int refCount;
600 
601     // indicates whether this connection's life cycle is managed (by us)
602     private boolean managed;
603 
604     private User user;
605 
606     /**
607      * Cluster registry of basic info such as clusterid and meta region location.
608      */
609      Registry registry;
610 
611      HConnectionImplementation(Configuration conf, boolean managed) throws IOException {
612        this(conf, managed, null, null);
613      }
614 
615     /**
616      * constructor
617      * @param conf Configuration object
618      * @param managed If true, does not do full shutdown on close; i.e. cleanup of connection
619      * to zk and shutdown of all services; we just close down the resources this connection was
620      * responsible for and decrement usage counters.  It is up to the caller to do the full
621      * cleanup.  It is set when we want have connection sharing going on -- reuse of zk connection,
622      * and cached region locations, established regionserver connections, etc.  When connections
623      * are shared, we have reference counting going on and will only do full cleanup when no more
624      * users of an HConnectionImplementation instance.
625      */
626     HConnectionImplementation(Configuration conf, boolean managed,
627         ExecutorService pool, User user) throws IOException {
628       this(conf);
629       this.user = user;
630       this.batchPool = pool;
631       this.managed = managed;
632       this.registry = setupRegistry();
633       retrieveClusterId();
634 
635       this.rpcClient = new RpcClient(this.conf, this.clusterId);
636 
637       // Do we publish the status?
638       boolean shouldListen = conf.getBoolean(HConstants.STATUS_PUBLISHED,
639           HConstants.STATUS_PUBLISHED_DEFAULT);
640       Class<? extends ClusterStatusListener.Listener> listenerClass =
641           conf.getClass(ClusterStatusListener.STATUS_LISTENER_CLASS,
642               ClusterStatusListener.DEFAULT_STATUS_LISTENER_CLASS,
643               ClusterStatusListener.Listener.class);
644       if (shouldListen) {
645         if (listenerClass == null) {
646           LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
647               ClusterStatusListener.STATUS_LISTENER_CLASS + " is not set - not listening status");
648         } else {
649           clusterStatusListener = new ClusterStatusListener(
650               new ClusterStatusListener.DeadServerHandler() {
651                 @Override
652                 public void newDead(ServerName sn) {
653                   clearCaches(sn);
654                   rpcClient.cancelConnections(sn.getHostname(), sn.getPort(),
655                       new SocketException(sn.getServerName() +
656                           " is dead: closing its connection."));
657                 }
658               }, conf, listenerClass);
659         }
660       }
661     }
662 
663 
664     /**
665      * For tests.
666      */
667     protected HConnectionImplementation(Configuration conf) {
668       this.conf = conf;
669       this.closed = false;
670       this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
671           HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
672       this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
673           HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
674       this.rpcTimeout = conf.getInt(
675           HConstants.HBASE_RPC_TIMEOUT_KEY,
676           HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
677       this.prefetchRegionLimit = conf.getInt(
678           HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
679           HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
680     }
681 
682     @Override
683     public HTableInterface getTable(String tableName) throws IOException {
684       return getTable(TableName.valueOf(tableName));
685     }
686 
687     @Override
688     public HTableInterface getTable(byte[] tableName) throws IOException {
689       return getTable(TableName.valueOf(tableName));
690     }
691 
692     @Override
693     public HTableInterface getTable(TableName tableName) throws IOException {
694       return getTable(tableName, getBatchPool());
695     }
696 
697     @Override
698     public HTableInterface getTable(String tableName, ExecutorService pool) throws IOException {
699       return getTable(TableName.valueOf(tableName), pool);
700     }
701 
702     @Override
703     public HTableInterface getTable(byte[] tableName, ExecutorService pool) throws IOException {
704       return getTable(TableName.valueOf(tableName), pool);
705     }
706 
707     @Override
708     public HTableInterface getTable(TableName tableName, ExecutorService pool) throws IOException {
709       if (managed) {
710         throw new IOException("The connection has to be unmanaged.");
711       }
712       return new HTable(tableName, this, pool);
713     }
714 
715     private ExecutorService getBatchPool() {
716       if (batchPool == null) {
717         // shared HTable thread executor not yet initialized
718         synchronized (this) {
719           if (batchPool == null) {
720             int maxThreads = conf.getInt("hbase.hconnection.threads.max", 256);
721             int coreThreads = conf.getInt("hbase.hconnection.threads.core", 0);
722             if (maxThreads == 0) {
723               maxThreads = Runtime.getRuntime().availableProcessors() * 8;
724             }
725             long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 10);
726             LinkedBlockingQueue<Runnable> workQueue =
727               new LinkedBlockingQueue<Runnable>(maxThreads *
728                 conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
729                   HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
730             this.batchPool = new ThreadPoolExecutor(
731                 coreThreads,
732                 maxThreads,
733                 keepAliveTime,
734                 TimeUnit.SECONDS,
735                 workQueue,
736                 Threads.newDaemonThreadFactory(toString() + "-shared-"));
737           }
738           this.cleanupPool = true;
739         }
740       }
741       return this.batchPool;
742     }
743 
744     protected ExecutorService getCurrentBatchPool() {
745       return batchPool;
746     }
747 
748     private void shutdownBatchPool() {
749       if (this.cleanupPool && this.batchPool != null && !this.batchPool.isShutdown()) {
750         this.batchPool.shutdown();
751         try {
752           if (!this.batchPool.awaitTermination(10, TimeUnit.SECONDS)) {
753             this.batchPool.shutdownNow();
754           }
755         } catch (InterruptedException e) {
756           this.batchPool.shutdownNow();
757         }
758       }
759     }
760 
761     /**
762      * @return The cluster registry implementation to use.
763      * @throws IOException
764      */
765     private Registry setupRegistry() throws IOException {
766       String registryClass = this.conf.get("hbase.client.registry.impl",
767         ZooKeeperRegistry.class.getName());
768       Registry registry = null;
769       try {
770         registry = (Registry)Class.forName(registryClass).newInstance();
771       } catch (Throwable t) {
772         throw new IOException(t);
773       }
774       registry.init(this);
775       return registry;
776     }
777 
778     /**
779      * For tests only.
780      * @param rpcClient Client we should use instead.
781      * @return Previous rpcClient
782      */
783     RpcClient setRpcClient(final RpcClient rpcClient) {
784       RpcClient oldRpcClient = this.rpcClient;
785       this.rpcClient = rpcClient;
786       return oldRpcClient;
787     }
788 
789     /**
790      * An identifier that will remain the same for a given connection.
791      * @return
792      */
793     public String toString(){
794       return "hconnection-0x" + Integer.toHexString(hashCode());
795     }
796 
797     protected String clusterId = null;
798 
799     void retrieveClusterId() {
800       if (clusterId != null) return;
801       this.clusterId = this.registry.getClusterId();
802       if (clusterId == null) {
803         clusterId = HConstants.CLUSTER_ID_DEFAULT;
804         LOG.debug("clusterid came back null, using default " + clusterId);
805       }
806     }
807 
808     @Override
809     public Configuration getConfiguration() {
810       return this.conf;
811     }
812 
813     private void checkIfBaseNodeAvailable(ZooKeeperWatcher zkw)
814       throws MasterNotRunningException {
815       String errorMsg;
816       try {
817         if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {
818           errorMsg = "The node " + zkw.baseZNode+" is not in ZooKeeper. "
819             + "It should have been written by the master. "
820             + "Check the value configured in 'zookeeper.znode.parent'. "
821             + "There could be a mismatch with the one configured in the master.";
822           LOG.error(errorMsg);
823           throw new MasterNotRunningException(errorMsg);
824         }
825       } catch (KeeperException e) {
826         errorMsg = "Can't get connection to ZooKeeper: " + e.getMessage();
827         LOG.error(errorMsg);
828         throw new MasterNotRunningException(errorMsg, e);
829       }
830     }
831 
832     /**
833      * @return true if the master is running, throws an exception otherwise
834      * @throws MasterNotRunningException - if the master is not running
835      * @throws ZooKeeperConnectionException
836      */
837     @Override
838     public boolean isMasterRunning()
839     throws MasterNotRunningException, ZooKeeperConnectionException {
840       // When getting the master connection, we check it's running,
841       // so if there is no exception, it means we've been able to get a
842       // connection on a running master
843       MasterKeepAliveConnection m = getKeepAliveMasterService();
844       m.close();
845       return true;
846     }
847 
848     @Override
849     public HRegionLocation getRegionLocation(final TableName tableName,
850         final byte [] row, boolean reload)
851     throws IOException {
852       return reload? relocateRegion(tableName, row): locateRegion(tableName, row);
853     }
854 
855     @Override
856     public HRegionLocation getRegionLocation(final byte[] tableName,
857         final byte [] row, boolean reload)
858     throws IOException {
859       return getRegionLocation(TableName.valueOf(tableName), row, reload);
860     }
861 
862     @Override
863     public boolean isTableEnabled(TableName tableName) throws IOException {
864       return this.registry.isTableOnlineState(tableName, true);
865     }
866 
867     @Override
868     public boolean isTableEnabled(byte[] tableName) throws IOException {
869       return isTableEnabled(TableName.valueOf(tableName));
870     }
871 
872     @Override
873     public boolean isTableDisabled(TableName tableName) throws IOException {
874       return this.registry.isTableOnlineState(tableName, false);
875     }
876 
877     @Override
878     public boolean isTableDisabled(byte[] tableName) throws IOException {
879       return isTableDisabled(TableName.valueOf(tableName));
880     }
881 
882     @Override
883     public boolean isTableAvailable(final TableName tableName) throws IOException {
884       final AtomicBoolean available = new AtomicBoolean(true);
885       final AtomicInteger regionCount = new AtomicInteger(0);
886       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
887         @Override
888         public boolean processRow(Result row) throws IOException {
889           HRegionInfo info = MetaScanner.getHRegionInfo(row);
890           if (info != null && !info.isSplitParent()) {
891             if (tableName.equals(info.getTable())) {
892               ServerName server = HRegionInfo.getServerName(row);
893               if (server == null) {
894                 available.set(false);
895                 return false;
896               }
897               regionCount.incrementAndGet();
898             } else if (tableName.compareTo(info.getTable()) < 0) {
899               // Return if we are done with the current table
900               return false;
901             }
902           }
903           return true;
904         }
905       };
906       MetaScanner.metaScan(conf, this, visitor, tableName);
907       return available.get() && (regionCount.get() > 0);
908     }
909 
910     @Override
911     public boolean isTableAvailable(final byte[] tableName) throws IOException {
912       return isTableAvailable(TableName.valueOf(tableName));
913     }
914 
915     @Override
916     public boolean isTableAvailable(final TableName tableName, final byte[][] splitKeys)
917         throws IOException {
918       final AtomicBoolean available = new AtomicBoolean(true);
919       final AtomicInteger regionCount = new AtomicInteger(0);
920       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
921         @Override
922         public boolean processRow(Result row) throws IOException {
923           HRegionInfo info = MetaScanner.getHRegionInfo(row);
924           if (info != null && !info.isSplitParent()) {
925             if (tableName.equals(info.getTable())) {
926               ServerName server = HRegionInfo.getServerName(row);
927               if (server == null) {
928                 available.set(false);
929                 return false;
930               }
931               if (!Bytes.equals(info.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
932                 for (byte[] splitKey : splitKeys) {
933                   // Just check if the splitkey is available
934                   if (Bytes.equals(info.getStartKey(), splitKey)) {
935                     regionCount.incrementAndGet();
936                     break;
937                   }
938                 }
939               } else {
940                 // Always empty start row should be counted
941                 regionCount.incrementAndGet();
942               }
943             } else if (tableName.compareTo(info.getTable()) < 0) {
944               // Return if we are done with the current table
945               return false;
946             }
947           }
948           return true;
949         }
950       };
951       MetaScanner.metaScan(conf, this, visitor, tableName);
952       // +1 needs to be added so that the empty start row is also taken into account
953       return available.get() && (regionCount.get() == splitKeys.length + 1);
954     }
955 
956     @Override
957     public boolean isTableAvailable(final byte[] tableName, final byte[][] splitKeys)
958         throws IOException {
959       return isTableAvailable(TableName.valueOf(tableName), splitKeys);
960     }
961 
962     @Override
963     public HRegionLocation locateRegion(final byte[] regionName) throws IOException {
964       return locateRegion(HRegionInfo.getTable(regionName),
965           HRegionInfo.getStartKey(regionName), false, true);
966     }
967 
968     @Override
969     public boolean isDeadServer(ServerName sn) {
970       if (clusterStatusListener == null) {
971         return false;
972       } else {
973         return clusterStatusListener.isDeadServer(sn);
974       }
975     }
976 
977     @Override
978     public List<HRegionLocation> locateRegions(final TableName tableName)
979     throws IOException {
980       return locateRegions (tableName, false, true);
981     }
982 
983     @Override
984     public List<HRegionLocation> locateRegions(final byte[] tableName)
985     throws IOException {
986       return locateRegions(TableName.valueOf(tableName));
987     }
988 
989     @Override
990     public List<HRegionLocation> locateRegions(final TableName tableName,
991         final boolean useCache, final boolean offlined) throws IOException {
992       NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, this,
993           tableName, offlined);
994       final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();
995       for (HRegionInfo regionInfo : regions.keySet()) {
996         locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true));
997       }
998       return locations;
999     }
1000 
1001     @Override
1002     public List<HRegionLocation> locateRegions(final byte[] tableName,
1003        final boolean useCache, final boolean offlined) throws IOException {
1004       return locateRegions(TableName.valueOf(tableName), useCache, offlined);
1005     }
1006 
1007     @Override
1008     public HRegionLocation locateRegion(final TableName tableName,
1009         final byte [] row)
1010     throws IOException{
1011       return locateRegion(tableName, row, true, true);
1012     }
1013 
1014     @Override
1015     public HRegionLocation locateRegion(final byte[] tableName,
1016         final byte [] row)
1017     throws IOException{
1018       return locateRegion(TableName.valueOf(tableName), row);
1019     }
1020 
1021     @Override
1022     public HRegionLocation relocateRegion(final TableName tableName,
1023         final byte [] row) throws IOException{
1024       // Since this is an explicit request not to use any caching, finding
1025       // disabled tables should not be desirable.  This will ensure that an exception is thrown when
1026       // the first time a disabled table is interacted with.
1027       if (isTableDisabled(tableName)) {
1028         throw new TableNotEnabledException(tableName.getNameAsString() + " is disabled.");
1029       }
1030 
1031       return locateRegion(tableName, row, false, true);
1032     }
1033 
1034     @Override
1035     public HRegionLocation relocateRegion(final byte[] tableName,
1036         final byte [] row) throws IOException {
1037       return relocateRegion(TableName.valueOf(tableName), row);
1038     }
1039 
1040 
1041     private HRegionLocation locateRegion(final TableName tableName,
1042       final byte [] row, boolean useCache, boolean retry)
1043     throws IOException {
1044       if (this.closed) throw new IOException(toString() + " closed");
1045       if (tableName== null || tableName.getName().length == 0) {
1046         throw new IllegalArgumentException(
1047             "table name cannot be null or zero length");
1048       }
1049 
1050       if (tableName.equals(TableName.META_TABLE_NAME)) {
1051         return this.registry.getMetaRegionLocation();
1052       } else {
1053         // Region not in the cache - have to go to the meta RS
1054         return locateRegionInMeta(TableName.META_TABLE_NAME, tableName, row,
1055           useCache, userRegionLock, retry);
1056       }
1057     }
1058 
1059     /*
1060      * Search hbase:meta for the HRegionLocation info that contains the table and
1061      * row we're seeking. It will prefetch certain number of regions info and
1062      * save them to the global region cache.
1063      */
1064     private void prefetchRegionCache(final TableName tableName,
1065         final byte[] row) {
1066       // Implement a new visitor for MetaScanner, and use it to walk through
1067       // the hbase:meta
1068       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
1069         public boolean processRow(Result result) throws IOException {
1070           try {
1071             HRegionInfo regionInfo = MetaScanner.getHRegionInfo(result);
1072             if (regionInfo == null) {
1073               return true;
1074             }
1075 
1076             // possible we got a region of a different table...
1077             if (!regionInfo.getTable().equals(tableName)) {
1078               return false; // stop scanning
1079             }
1080             if (regionInfo.isOffline()) {
1081               // don't cache offline regions
1082               return true;
1083             }
1084 
1085             ServerName serverName = HRegionInfo.getServerName(result);
1086             if (serverName == null) {
1087               return true; // don't cache it
1088             }
1089             // instantiate the location
1090             long seqNum = HRegionInfo.getSeqNumDuringOpen(result);
1091             HRegionLocation loc = new HRegionLocation(regionInfo, serverName, seqNum);
1092             // cache this meta entry
1093             cacheLocation(tableName, null, loc);
1094             return true;
1095           } catch (RuntimeException e) {
1096             throw new IOException(e);
1097           }
1098         }
1099       };
1100       try {
1101         // pre-fetch certain number of regions info at region cache.
1102         MetaScanner.metaScan(conf, this, visitor, tableName, row,
1103             this.prefetchRegionLimit, TableName.META_TABLE_NAME);
1104       } catch (IOException e) {
1105         LOG.warn("Encountered problems when prefetch hbase:meta table: ", e);
1106       }
1107     }
1108 
1109     /*
1110       * Search the hbase:meta table for the HRegionLocation
1111       * info that contains the table and row we're seeking.
1112       */
1113     private HRegionLocation locateRegionInMeta(final TableName parentTable,
1114       final TableName tableName, final byte [] row, boolean useCache,
1115       Object regionLockObject, boolean retry)
1116     throws IOException {
1117       HRegionLocation location;
1118       // If we are supposed to be using the cache, look in the cache to see if
1119       // we already have the region.
1120       if (useCache) {
1121         location = getCachedLocation(tableName, row);
1122         if (location != null) {
1123           return location;
1124         }
1125       }
1126       int localNumRetries = retry ? numTries : 1;
1127       // build the key of the meta region we should be looking for.
1128       // the extra 9's on the end are necessary to allow "exact" matches
1129       // without knowing the precise region names.
1130       byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
1131         HConstants.NINES, false);
1132       for (int tries = 0; true; tries++) {
1133         if (tries >= localNumRetries) {
1134           throw new NoServerForRegionException("Unable to find region for "
1135             + Bytes.toStringBinary(row) + " after " + numTries + " tries.");
1136         }
1137 
1138         HRegionLocation metaLocation = null;
1139         try {
1140           // locate the meta region
1141           metaLocation = locateRegion(parentTable, metaKey, true, false);
1142           // If null still, go around again.
1143           if (metaLocation == null) continue;
1144           ClientService.BlockingInterface service = getClient(metaLocation.getServerName());
1145 
1146           Result regionInfoRow;
1147           // This block guards against two threads trying to load the meta
1148           // region at the same time. The first will load the meta region and
1149           // the second will use the value that the first one found.
1150           if (useCache) {
1151             if (TableName.META_TABLE_NAME.equals(parentTable) &&
1152                 getRegionCachePrefetch(tableName)) {
1153               synchronized (regionLockObject) {
1154                 // Check the cache again for a hit in case some other thread made the
1155                 // same query while we were waiting on the lock.
1156                 location = getCachedLocation(tableName, row);
1157                 if (location != null) {
1158                   return location;
1159                 }
1160                 // If the parent table is META, we may want to pre-fetch some
1161                 // region info into the global region cache for this table.
1162                 prefetchRegionCache(tableName, row);
1163               }
1164             }
1165             location = getCachedLocation(tableName, row);
1166             if (location != null) {
1167               return location;
1168             }
1169           } else {
1170             // If we are not supposed to be using the cache, delete any existing cached location
1171             // so it won't interfere.
1172             forceDeleteCachedLocation(tableName, row);
1173           }
1174 
1175           // Query the meta region for the location of the meta region
1176           regionInfoRow = ProtobufUtil.getRowOrBefore(service,
1177               metaLocation.getRegionInfo().getRegionName(), metaKey,
1178               HConstants.CATALOG_FAMILY);
1179 
1180           if (regionInfoRow == null) {
1181             throw new TableNotFoundException(tableName);
1182           }
1183 
1184           // convert the row result into the HRegionLocation we need!
1185           HRegionInfo regionInfo = MetaScanner.getHRegionInfo(regionInfoRow);
1186           if (regionInfo == null) {
1187             throw new IOException("HRegionInfo was null or empty in " +
1188               parentTable + ", row=" + regionInfoRow);
1189           }
1190 
1191           // possible we got a region of a different table...
1192           if (!regionInfo.getTable().equals(tableName)) {
1193             throw new TableNotFoundException(
1194                   "Table '" + tableName + "' was not found, got: " +
1195                   regionInfo.getTable() + ".");
1196           }
1197           if (regionInfo.isSplit()) {
1198             throw new RegionOfflineException("the only available region for" +
1199               " the required row is a split parent," +
1200               " the daughters should be online soon: " +
1201               regionInfo.getRegionNameAsString());
1202           }
1203           if (regionInfo.isOffline()) {
1204             throw new RegionOfflineException("the region is offline, could" +
1205               " be caused by a disable table call: " +
1206               regionInfo.getRegionNameAsString());
1207           }
1208 
1209           ServerName serverName = HRegionInfo.getServerName(regionInfoRow);
1210           if (serverName == null) {
1211             throw new NoServerForRegionException("No serverName " +
1212               "in " + parentTable + " for " +
1213               regionInfo.getRegionNameAsString() + " containing " +
1214               Bytes.toStringBinary(row));
1215           }
1216 
1217           if (isDeadServer(serverName)){
1218             throw new RegionServerStoppedException("hbase:meta says the region "+
1219                 regionInfo.getRegionNameAsString()+" is managed by the server " + serverName +
1220                 ", but it is dead.");
1221           }
1222 
1223           // Instantiate the location
1224           location = new HRegionLocation(regionInfo, serverName,
1225             HRegionInfo.getSeqNumDuringOpen(regionInfoRow));
1226           cacheLocation(tableName, null, location);
1227           return location;
1228         } catch (TableNotFoundException e) {
1229           // if we got this error, probably means the table just plain doesn't
1230           // exist. rethrow the error immediately. this should always be coming
1231           // from the HTable constructor.
1232           throw e;
1233         } catch (IOException e) {
1234           if (e instanceof RemoteException) {
1235             e = ((RemoteException)e).unwrapRemoteException();
1236           }
1237           if (tries < numTries - 1) {
1238             if (LOG.isDebugEnabled()) {
1239               LOG.debug("locateRegionInMeta failed; parentTable=" + parentTable +
1240                 ", metaLocation=" + ((metaLocation == null)? "null": "{" + metaLocation + "}") +
1241                 ", attempt=" + tries + "/" + this.numTries + "; retrying after=" +
1242                 ConnectionUtils.getPauseTime(this.pause, tries) + "ms; because: " + e.getMessage());
1243             }
1244           } else {
1245             throw e;
1246           }
1247           // Only relocate the parent region if necessary
1248           if(!(e instanceof RegionOfflineException ||
1249               e instanceof NoServerForRegionException)) {
1250             relocateRegion(parentTable, metaKey);
1251           }
1252         }
1253         try{
1254           Thread.sleep(ConnectionUtils.getPauseTime(this.pause, tries));
1255         } catch (InterruptedException e) {
1256           Thread.currentThread().interrupt();
1257           throw new IOException("Giving up trying to location region in " +
1258             "meta: thread is interrupted.");
1259         }
1260       }
1261     }
1262 
1263     /*
1264      * Search the cache for a location that fits our table and row key.
1265      * Return null if no suitable region is located.
1266      *
1267      * @param tableName
1268      * @param row
1269      * @return Null or region location found in cache.
1270      */
1271     HRegionLocation getCachedLocation(final TableName tableName,
1272         final byte [] row) {
1273       ConcurrentSkipListMap<byte[], HRegionLocation> tableLocations =
1274         getTableLocations(tableName);
1275 
1276       Entry<byte[], HRegionLocation> e = tableLocations.floorEntry(row);
1277       if (e == null) {
1278         return null;
1279       }
1280       HRegionLocation possibleRegion = e.getValue();
1281 
1282       // make sure that the end key is greater than the row we're looking
1283       // for, otherwise the row actually belongs in the next region, not
1284       // this one. the exception case is when the endkey is
1285       // HConstants.EMPTY_END_ROW, signifying that the region we're
1286       // checking is actually the last region in the table.
1287       byte[] endKey = possibleRegion.getRegionInfo().getEndKey();
1288       if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
1289           tableName.getRowComparator().compareRows(
1290               endKey, 0, endKey.length, row, 0, row.length) > 0) {
1291         return possibleRegion;
1292       }
1293 
1294       // Passed all the way through, so we got nothing - complete cache miss
1295       return null;
1296     }
1297 
1298     /**
1299      * Delete a cached location, no matter what it is. Called when we were told to not use cache.
1300      * @param tableName tableName
1301      * @param row
1302      */
1303     void forceDeleteCachedLocation(final TableName tableName, final byte [] row) {
1304       HRegionLocation rl = null;
1305       Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1306       // start to examine the cache. we can only do cache actions
1307       // if there's something in the cache for this table.
1308       rl = getCachedLocation(tableName, row);
1309       if (rl != null) {
1310         tableLocations.remove(rl.getRegionInfo().getStartKey());
1311       }
1312       if ((rl != null) && LOG.isDebugEnabled()) {
1313         LOG.debug("Removed " + rl.getHostname() + ":" + rl.getPort()
1314           + " as a location of " + rl.getRegionInfo().getRegionNameAsString() +
1315           " for tableName=" + tableName + " from cache");
1316       }
1317     }
1318 
1319     /*
1320      * Delete all cached entries of a table that maps to a specific location.
1321      */
1322     @Override
1323     public void clearCaches(final ServerName serverName) {
1324       if (!this.cachedServers.contains(serverName)) {
1325         return;
1326       }
1327 
1328       boolean deletedSomething = false;
1329       synchronized (this.cachedServers) {
1330         // We block here, because if there is an error on a server, it's likely that multiple
1331         //  threads will get the error  simultaneously. If there are hundreds of thousand of
1332         //  region location to check, it's better to do this only once. A better pattern would
1333         //  be to check if the server is dead when we get the region location.
1334         if (!this.cachedServers.contains(serverName)) {
1335           return;
1336         }
1337         for (Map<byte[], HRegionLocation> tableLocations : cachedRegionLocations.values()) {
1338           for (Entry<byte[], HRegionLocation> e : tableLocations.entrySet()) {
1339             HRegionLocation value = e.getValue();
1340             if (value != null
1341                 && serverName.equals(value.getServerName())) {
1342               tableLocations.remove(e.getKey());
1343               deletedSomething = true;
1344             }
1345           }
1346         }
1347         this.cachedServers.remove(serverName);
1348       }
1349       if (deletedSomething && LOG.isDebugEnabled()) {
1350         LOG.debug("Removed all cached region locations that map to " + serverName);
1351       }
1352     }
1353 
1354     /*
1355      * @param tableName
1356      * @return Map of cached locations for passed <code>tableName</code>
1357      */
1358     private ConcurrentSkipListMap<byte[], HRegionLocation> getTableLocations(
1359         final TableName tableName) {
1360       // find the map of cached locations for this table
1361       ConcurrentSkipListMap<byte[], HRegionLocation> result;
1362       result = this.cachedRegionLocations.get(tableName);
1363       // if tableLocations for this table isn't built yet, make one
1364       if (result == null) {
1365         result = new ConcurrentSkipListMap<byte[], HRegionLocation>(Bytes.BYTES_COMPARATOR);
1366         ConcurrentSkipListMap<byte[], HRegionLocation> old =
1367             this.cachedRegionLocations.putIfAbsent(tableName, result);
1368         if (old != null) {
1369           return old;
1370         }
1371       }
1372       return result;
1373     }
1374 
1375     @Override
1376     public void clearRegionCache() {
1377       this.cachedRegionLocations.clear();
1378       this.cachedServers.clear();
1379     }
1380 
1381     @Override
1382     public void clearRegionCache(final TableName tableName) {
1383       this.cachedRegionLocations.remove(tableName);
1384     }
1385 
1386     @Override
1387     public void clearRegionCache(final byte[] tableName) {
1388       clearRegionCache(TableName.valueOf(tableName));
1389     }
1390 
1391     /**
1392      * Put a newly discovered HRegionLocation into the cache.
1393      * @param tableName The table name.
1394      * @param source the source of the new location, if it's not coming from meta
1395      * @param location the new location
1396      */
1397     private void cacheLocation(final TableName tableName, final HRegionLocation source,
1398         final HRegionLocation location) {
1399       boolean isFromMeta = (source == null);
1400       byte [] startKey = location.getRegionInfo().getStartKey();
1401       ConcurrentMap<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1402       HRegionLocation oldLocation = tableLocations.putIfAbsent(startKey, location);
1403       boolean isNewCacheEntry = (oldLocation == null);
1404       if (isNewCacheEntry) {
1405         cachedServers.add(location.getServerName());
1406         return;
1407       }
1408       boolean updateCache;
1409       // If the server in cache sends us a redirect, assume it's always valid.
1410       if (oldLocation.equals(source)) {
1411         updateCache = true;
1412       } else {
1413         long newLocationSeqNum = location.getSeqNum();
1414         // Meta record is stale - some (probably the same) server has closed the region
1415         // with later seqNum and told us about the new location.
1416         boolean isStaleMetaRecord = isFromMeta && (oldLocation.getSeqNum() > newLocationSeqNum);
1417         // Same as above for redirect. However, in this case, if the number is equal to previous
1418         // record, the most common case is that first the region was closed with seqNum, and then
1419         // opened with the same seqNum; hence we will ignore the redirect.
1420         // There are so many corner cases with various combinations of opens and closes that
1421         // an additional counter on top of seqNum would be necessary to handle them all.
1422         boolean isStaleRedirect = !isFromMeta && (oldLocation.getSeqNum() >= newLocationSeqNum);
1423         boolean isStaleUpdate = (isStaleMetaRecord || isStaleRedirect);
1424         updateCache = (!isStaleUpdate);
1425       }
1426       if (updateCache) {
1427         tableLocations.replace(startKey, oldLocation, location);
1428         cachedServers.add(location.getServerName());
1429       }
1430     }
1431 
1432     // Map keyed by service name + regionserver to service stub implementation
1433     private final ConcurrentHashMap<String, Object> stubs =
1434       new ConcurrentHashMap<String, Object>();
1435     // Map of locks used creating service stubs per regionserver.
1436     private final ConcurrentHashMap<String, String> connectionLock =
1437       new ConcurrentHashMap<String, String>();
1438 
1439     /**
1440      * State of the MasterService connection/setup.
1441      */
1442     static class MasterServiceState {
1443       HConnection connection;
1444       MasterService.BlockingInterface stub;
1445       int userCount;
1446       long keepAliveUntil = Long.MAX_VALUE;
1447 
1448       MasterServiceState (final HConnection connection) {
1449         super();
1450         this.connection = connection;
1451       }
1452 
1453       @Override
1454       public String toString() {
1455         return "MasterService";
1456       }
1457 
1458       Object getStub() {
1459         return this.stub;
1460       }
1461 
1462       void clearStub() {
1463         this.stub = null;
1464       }
1465 
1466       boolean isMasterRunning() throws ServiceException {
1467         IsMasterRunningResponse response =
1468           this.stub.isMasterRunning(null, RequestConverter.buildIsMasterRunningRequest());
1469         return response != null? response.getIsMasterRunning(): false;
1470       }
1471     }
1472 
1473     /**
1474      * Makes a client-side stub for master services. Sub-class to specialize.
1475      * Depends on hosting class so not static.  Exists so we avoid duplicating a bunch of code
1476      * when setting up the MasterMonitorService and MasterAdminService.
1477      */
1478     abstract class StubMaker {
1479       /**
1480        * Returns the name of the service stub being created.
1481        */
1482       protected abstract String getServiceName();
1483 
1484       /**
1485        * Make stub and cache it internal so can be used later doing the isMasterRunning call.
1486        * @param channel
1487        */
1488       protected abstract Object makeStub(final BlockingRpcChannel channel);
1489 
1490       /**
1491        * Once setup, check it works by doing isMasterRunning check.
1492        * @throws ServiceException
1493        */
1494       protected abstract void isMasterRunning() throws ServiceException;
1495 
1496       /**
1497        * Create a stub. Try once only.  It is not typed because there is no common type to
1498        * protobuf services nor their interfaces.  Let the caller do appropriate casting.
1499        * @return A stub for master services.
1500        * @throws IOException
1501        * @throws KeeperException
1502        * @throws ServiceException
1503        */
1504       private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
1505         ZooKeeperKeepAliveConnection zkw;
1506         try {
1507           zkw = getKeepAliveZooKeeperWatcher();
1508         } catch (IOException e) {
1509           throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
1510         }
1511         try {
1512           checkIfBaseNodeAvailable(zkw);
1513           ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
1514           if (sn == null) {
1515             String msg = "ZooKeeper available but no active master location found";
1516             LOG.info(msg);
1517             throw new MasterNotRunningException(msg);
1518           }
1519           if (isDeadServer(sn)) {
1520             throw new MasterNotRunningException(sn + " is dead.");
1521           }
1522           // Use the security info interface name as our stub key
1523           String key = getStubKey(getServiceName(), sn.getHostAndPort());
1524           connectionLock.putIfAbsent(key, key);
1525           Object stub = null;
1526           synchronized (connectionLock.get(key)) {
1527             stub = stubs.get(key);
1528             if (stub == null) {
1529               BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn,
1530                 user, rpcTimeout);
1531               stub = makeStub(channel);
1532               isMasterRunning();
1533               stubs.put(key, stub);
1534             }
1535           }
1536           return stub;
1537         } finally {
1538           zkw.close();
1539         }
1540       }
1541 
1542       /**
1543        * Create a stub against the master.  Retry if necessary.
1544        * @return A stub to do <code>intf</code> against the master
1545        * @throws MasterNotRunningException
1546        */
1547       @edu.umd.cs.findbugs.annotations.SuppressWarnings (value="SWL_SLEEP_WITH_LOCK_HELD")
1548       Object makeStub() throws MasterNotRunningException {
1549         // The lock must be at the beginning to prevent multiple master creations
1550         //  (and leaks) in a multithread context
1551         synchronized (masterAndZKLock) {
1552           Exception exceptionCaught = null;
1553           Object stub = null;
1554           int tries = 0;
1555           while (!closed && stub == null) {
1556             tries++;
1557             try {
1558               stub = makeStubNoRetries();
1559             } catch (IOException e) {
1560               exceptionCaught = e;
1561             } catch (KeeperException e) {
1562               exceptionCaught = e;
1563             } catch (ServiceException e) {
1564               exceptionCaught = e;
1565             }
1566 
1567             if (exceptionCaught != null)
1568               // It failed. If it's not the last try, we're going to wait a little
1569               if (tries < numTries) {
1570                 // tries at this point is 1 or more; decrement to start from 0.
1571                 long pauseTime = ConnectionUtils.getPauseTime(pause, tries - 1);
1572                 LOG.info("getMaster attempt " + tries + " of " + numTries +
1573                     " failed; retrying after sleep of " + pauseTime + ", exception=" +
1574                   exceptionCaught);
1575 
1576                 try {
1577                   Thread.sleep(pauseTime);
1578                 } catch (InterruptedException e) {
1579                   Thread.currentThread().interrupt();
1580                   throw new RuntimeException(
1581                       "Thread was interrupted while trying to connect to master.", e);
1582                 }
1583               } else {
1584                 // Enough tries, we stop now
1585                 LOG.info("getMaster attempt " + tries + " of " + numTries +
1586                     " failed; no more retrying.", exceptionCaught);
1587                 throw new MasterNotRunningException(exceptionCaught);
1588               }
1589           }
1590 
1591           if (stub == null) {
1592             // implies this.closed true
1593             throw new MasterNotRunningException("Connection was closed while trying to get master");
1594           }
1595           return stub;
1596         }
1597       }
1598     }
1599 
1600     /**
1601      * Class to make a MasterServiceStubMaker stub.
1602      */
1603     class MasterServiceStubMaker extends StubMaker {
1604       private MasterService.BlockingInterface stub;
1605       @Override
1606       protected String getServiceName() {
1607         return MasterService.getDescriptor().getName();
1608       }
1609 
1610       @Override
1611       @edu.umd.cs.findbugs.annotations.SuppressWarnings("SWL_SLEEP_WITH_LOCK_HELD")
1612       MasterService.BlockingInterface makeStub() throws MasterNotRunningException {
1613         return (MasterService.BlockingInterface)super.makeStub();
1614       }
1615 
1616       @Override
1617       protected Object makeStub(BlockingRpcChannel channel) {
1618         this.stub = MasterService.newBlockingStub(channel);
1619         return this.stub;
1620       }
1621 
1622       @Override
1623       protected void isMasterRunning() throws ServiceException {
1624         this.stub.isMasterRunning(null, RequestConverter.buildIsMasterRunningRequest());
1625       }
1626     }
1627 
1628     @Override
1629     public AdminService.BlockingInterface getAdmin(final ServerName serverName)
1630         throws IOException {
1631       return getAdmin(serverName, false);
1632     }
1633 
1634     @Override
1635     // Nothing is done w/ the 'master' parameter.  It is ignored.
1636     public AdminService.BlockingInterface getAdmin(final ServerName serverName,
1637       final boolean master)
1638     throws IOException {
1639       if (isDeadServer(serverName)) {
1640         throw new RegionServerStoppedException(serverName + " is dead.");
1641       }
1642       String key = getStubKey(AdminService.BlockingInterface.class.getName(),
1643         serverName.getHostAndPort());
1644       this.connectionLock.putIfAbsent(key, key);
1645       AdminService.BlockingInterface stub = null;
1646       synchronized (this.connectionLock.get(key)) {
1647         stub = (AdminService.BlockingInterface)this.stubs.get(key);
1648         if (stub == null) {
1649           BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName,
1650             user, this.rpcTimeout);
1651           stub = AdminService.newBlockingStub(channel);
1652           this.stubs.put(key, stub);
1653         }
1654       }
1655       return stub;
1656     }
1657 
1658     @Override
1659     public ClientService.BlockingInterface getClient(final ServerName sn)
1660     throws IOException {
1661       if (isDeadServer(sn)) {
1662         throw new RegionServerStoppedException(sn + " is dead.");
1663       }
1664       String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort());
1665       this.connectionLock.putIfAbsent(key, key);
1666       ClientService.BlockingInterface stub = null;
1667       synchronized (this.connectionLock.get(key)) {
1668         stub = (ClientService.BlockingInterface)this.stubs.get(key);
1669         if (stub == null) {
1670           BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(sn,
1671             user, this.rpcTimeout);
1672           stub = ClientService.newBlockingStub(channel);
1673           // In old days, after getting stub/proxy, we'd make a call.  We are not doing that here.
1674           // Just fail on first actual call rather than in here on setup.
1675           this.stubs.put(key, stub);
1676         }
1677       }
1678       return stub;
1679     }
1680 
1681     static String getStubKey(final String serviceName, final String rsHostnamePort) {
1682       return serviceName + "@" + rsHostnamePort;
1683     }
1684 
1685     private ZooKeeperKeepAliveConnection keepAliveZookeeper;
1686     private int keepAliveZookeeperUserCount;
1687     private boolean canCloseZKW = true;
1688 
1689     // keepAlive time, in ms. No reason to make it configurable.
1690     private static final long keepAlive = 5 * 60 * 1000;
1691 
1692     /**
1693      * Retrieve a shared ZooKeeperWatcher. You must close it it once you've have finished with it.
1694      * @return The shared instance. Never returns null.
1695      */
1696     ZooKeeperKeepAliveConnection getKeepAliveZooKeeperWatcher()
1697       throws IOException {
1698       synchronized (masterAndZKLock) {
1699         if (keepAliveZookeeper == null) {
1700           if (this.closed) {
1701             throw new IOException(toString() + " closed");
1702           }
1703           // We don't check that our link to ZooKeeper is still valid
1704           // But there is a retry mechanism in the ZooKeeperWatcher itself
1705           keepAliveZookeeper = new ZooKeeperKeepAliveConnection(conf, this.toString(), this);
1706         }
1707         keepAliveZookeeperUserCount++;
1708         keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1709         return keepAliveZookeeper;
1710       }
1711     }
1712 
1713     void releaseZooKeeperWatcher(final ZooKeeperWatcher zkw) {
1714       if (zkw == null){
1715         return;
1716       }
1717       synchronized (masterAndZKLock) {
1718         --keepAliveZookeeperUserCount;
1719         if (keepAliveZookeeperUserCount <= 0 ){
1720           keepZooKeeperWatcherAliveUntil = System.currentTimeMillis() + keepAlive;
1721         }
1722       }
1723     }
1724 
1725     /**
1726      * Creates a Chore thread to check the connections to master & zookeeper
1727      *  and close them when they reach their closing time (
1728      *  {@link MasterServiceState#keepAliveUntil} and
1729      *  {@link #keepZooKeeperWatcherAliveUntil}). Keep alive time is
1730      *  managed by the release functions and the variable {@link #keepAlive}
1731      */
1732     private static class DelayedClosing extends Chore implements Stoppable {
1733       private HConnectionImplementation hci;
1734       Stoppable stoppable;
1735 
1736       private DelayedClosing(
1737         HConnectionImplementation hci, Stoppable stoppable){
1738         super(
1739           "ZooKeeperWatcher and Master delayed closing for connection "+hci,
1740           60*1000, // We check every minutes
1741           stoppable);
1742         this.hci = hci;
1743         this.stoppable = stoppable;
1744       }
1745 
1746       static DelayedClosing createAndStart(HConnectionImplementation hci){
1747         Stoppable stoppable = new Stoppable() {
1748               private volatile boolean isStopped = false;
1749               @Override public void stop(String why) { isStopped = true;}
1750               @Override public boolean isStopped() {return isStopped;}
1751             };
1752 
1753         return new DelayedClosing(hci, stoppable);
1754       }
1755 
1756       protected void closeMasterProtocol(MasterServiceState protocolState) {
1757         if (System.currentTimeMillis() > protocolState.keepAliveUntil) {
1758           hci.closeMasterService(protocolState);
1759           protocolState.keepAliveUntil = Long.MAX_VALUE;
1760         }
1761       }
1762 
1763       @Override
1764       protected void chore() {
1765         synchronized (hci.masterAndZKLock) {
1766           if (hci.canCloseZKW) {
1767             if (System.currentTimeMillis() >
1768               hci.keepZooKeeperWatcherAliveUntil) {
1769 
1770               hci.closeZooKeeperWatcher();
1771               hci.keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1772             }
1773           }
1774           closeMasterProtocol(hci.masterServiceState);
1775           closeMasterProtocol(hci.masterServiceState);
1776         }
1777       }
1778 
1779       @Override
1780       public void stop(String why) {
1781         stoppable.stop(why);
1782       }
1783 
1784       @Override
1785       public boolean isStopped() {
1786         return stoppable.isStopped();
1787       }
1788     }
1789 
1790     private void closeZooKeeperWatcher() {
1791       synchronized (masterAndZKLock) {
1792         if (keepAliveZookeeper != null) {
1793           LOG.info("Closing zookeeper sessionid=0x" +
1794             Long.toHexString(
1795               keepAliveZookeeper.getRecoverableZooKeeper().getSessionId()));
1796           keepAliveZookeeper.internalClose();
1797           keepAliveZookeeper = null;
1798         }
1799         keepAliveZookeeperUserCount = 0;
1800       }
1801     }
1802 
1803     final MasterServiceState masterServiceState = new MasterServiceState(this);
1804 
1805     @Override
1806     public MasterService.BlockingInterface getMaster() throws MasterNotRunningException {
1807       return getKeepAliveMasterService();
1808     }
1809 
1810     private void resetMasterServiceState(final MasterServiceState mss) {
1811       mss.userCount++;
1812       mss.keepAliveUntil = Long.MAX_VALUE;
1813     }
1814 
1815     @Override
1816     public MasterKeepAliveConnection getKeepAliveMasterService()
1817     throws MasterNotRunningException {
1818       synchronized (masterAndZKLock) {
1819         if (!isKeepAliveMasterConnectedAndRunning(this.masterServiceState)) {
1820           MasterServiceStubMaker stubMaker = new MasterServiceStubMaker();
1821           this.masterServiceState.stub = stubMaker.makeStub();
1822         }
1823         resetMasterServiceState(this.masterServiceState);
1824       }
1825       // Ugly delegation just so we can add in a Close method.
1826       final MasterService.BlockingInterface stub = this.masterServiceState.stub;
1827       return new MasterKeepAliveConnection() {
1828         MasterServiceState mss = masterServiceState;
1829         @Override
1830         public AddColumnResponse addColumn(RpcController controller, AddColumnRequest request)
1831         throws ServiceException {
1832           return stub.addColumn(controller, request);
1833         }
1834 
1835         @Override
1836         public DeleteColumnResponse deleteColumn(RpcController controller,
1837             DeleteColumnRequest request)
1838         throws ServiceException {
1839           return stub.deleteColumn(controller, request);
1840         }
1841 
1842         @Override
1843         public ModifyColumnResponse modifyColumn(RpcController controller,
1844             ModifyColumnRequest request)
1845         throws ServiceException {
1846           return stub.modifyColumn(controller, request);
1847         }
1848 
1849         @Override
1850         public MoveRegionResponse moveRegion(RpcController controller,
1851             MoveRegionRequest request) throws ServiceException {
1852           return stub.moveRegion(controller, request);
1853         }
1854 
1855         @Override
1856         public DispatchMergingRegionsResponse dispatchMergingRegions(
1857             RpcController controller, DispatchMergingRegionsRequest request)
1858             throws ServiceException {
1859           return stub.dispatchMergingRegions(controller, request);
1860         }
1861 
1862         @Override
1863         public AssignRegionResponse assignRegion(RpcController controller,
1864             AssignRegionRequest request) throws ServiceException {
1865           return stub.assignRegion(controller, request);
1866         }
1867 
1868         @Override
1869         public UnassignRegionResponse unassignRegion(RpcController controller,
1870             UnassignRegionRequest request) throws ServiceException {
1871           return stub.unassignRegion(controller, request);
1872         }
1873 
1874         @Override
1875         public OfflineRegionResponse offlineRegion(RpcController controller,
1876             OfflineRegionRequest request) throws ServiceException {
1877           return stub.offlineRegion(controller, request);
1878         }
1879 
1880         @Override
1881         public DeleteTableResponse deleteTable(RpcController controller,
1882             DeleteTableRequest request) throws ServiceException {
1883           return stub.deleteTable(controller, request);
1884         }
1885 
1886         @Override
1887         public EnableTableResponse enableTable(RpcController controller,
1888             EnableTableRequest request) throws ServiceException {
1889           return stub.enableTable(controller, request);
1890         }
1891 
1892         @Override
1893         public DisableTableResponse disableTable(RpcController controller,
1894             DisableTableRequest request) throws ServiceException {
1895           return stub.disableTable(controller, request);
1896         }
1897 
1898         @Override
1899         public ModifyTableResponse modifyTable(RpcController controller,
1900             ModifyTableRequest request) throws ServiceException {
1901           return stub.modifyTable(controller, request);
1902         }
1903 
1904         @Override
1905         public CreateTableResponse createTable(RpcController controller,
1906             CreateTableRequest request) throws ServiceException {
1907           return stub.createTable(controller, request);
1908         }
1909 
1910         @Override
1911         public ShutdownResponse shutdown(RpcController controller,
1912             ShutdownRequest request) throws ServiceException {
1913           return stub.shutdown(controller, request);
1914         }
1915 
1916         @Override
1917         public StopMasterResponse stopMaster(RpcController controller,
1918             StopMasterRequest request) throws ServiceException {
1919           return stub.stopMaster(controller, request);
1920         }
1921 
1922         @Override
1923         public BalanceResponse balance(RpcController controller,
1924             BalanceRequest request) throws ServiceException {
1925           return stub.balance(controller, request);
1926         }
1927 
1928         @Override
1929         public SetBalancerRunningResponse setBalancerRunning(
1930             RpcController controller, SetBalancerRunningRequest request)
1931             throws ServiceException {
1932           return stub.setBalancerRunning(controller, request);
1933         }
1934 
1935         @Override
1936         public RunCatalogScanResponse runCatalogScan(RpcController controller,
1937             RunCatalogScanRequest request) throws ServiceException {
1938           return stub.runCatalogScan(controller, request);
1939         }
1940 
1941         @Override
1942         public EnableCatalogJanitorResponse enableCatalogJanitor(
1943             RpcController controller, EnableCatalogJanitorRequest request)
1944             throws ServiceException {
1945           return stub.enableCatalogJanitor(controller, request);
1946         }
1947 
1948         @Override
1949         public IsCatalogJanitorEnabledResponse isCatalogJanitorEnabled(
1950             RpcController controller, IsCatalogJanitorEnabledRequest request)
1951             throws ServiceException {
1952           return stub.isCatalogJanitorEnabled(controller, request);
1953         }
1954 
1955         @Override
1956         public CoprocessorServiceResponse execMasterService(
1957             RpcController controller, CoprocessorServiceRequest request)
1958             throws ServiceException {
1959           return stub.execMasterService(controller, request);
1960         }
1961 
1962         @Override
1963         public SnapshotResponse snapshot(RpcController controller,
1964             SnapshotRequest request) throws ServiceException {
1965           return stub.snapshot(controller, request);
1966         }
1967 
1968         @Override
1969         public GetCompletedSnapshotsResponse getCompletedSnapshots(
1970             RpcController controller, GetCompletedSnapshotsRequest request)
1971             throws ServiceException {
1972           return stub.getCompletedSnapshots(controller, request);
1973         }
1974 
1975         @Override
1976         public DeleteSnapshotResponse deleteSnapshot(RpcController controller,
1977             DeleteSnapshotRequest request) throws ServiceException {
1978           return stub.deleteSnapshot(controller, request);
1979         }
1980 
1981         @Override
1982         public IsSnapshotDoneResponse isSnapshotDone(RpcController controller,
1983             IsSnapshotDoneRequest request) throws ServiceException {
1984           return stub.isSnapshotDone(controller, request);
1985         }
1986 
1987         @Override
1988         public RestoreSnapshotResponse restoreSnapshot(
1989             RpcController controller, RestoreSnapshotRequest request)
1990             throws ServiceException {
1991           return stub.restoreSnapshot(controller, request);
1992         }
1993 
1994         @Override
1995         public IsRestoreSnapshotDoneResponse isRestoreSnapshotDone(
1996             RpcController controller, IsRestoreSnapshotDoneRequest request)
1997             throws ServiceException {
1998           return stub.isRestoreSnapshotDone(controller, request);
1999         }
2000 
2001         @Override
2002         public IsMasterRunningResponse isMasterRunning(
2003             RpcController controller, IsMasterRunningRequest request)
2004             throws ServiceException {
2005           return stub.isMasterRunning(controller, request);
2006         }
2007 
2008         @Override
2009         public ModifyNamespaceResponse modifyNamespace(RpcController controller,
2010             ModifyNamespaceRequest request)
2011         throws ServiceException {
2012           return stub.modifyNamespace(controller, request);
2013         }
2014 
2015         @Override
2016         public CreateNamespaceResponse createNamespace(RpcController controller, CreateNamespaceRequest request) throws ServiceException {
2017           return stub.createNamespace(controller, request);
2018         }
2019 
2020         @Override
2021         public DeleteNamespaceResponse deleteNamespace(RpcController controller, DeleteNamespaceRequest request) throws ServiceException {
2022           return stub.deleteNamespace(controller, request);
2023         }
2024 
2025         @Override
2026         public GetNamespaceDescriptorResponse getNamespaceDescriptor(RpcController controller, GetNamespaceDescriptorRequest request) throws ServiceException {
2027           return stub.getNamespaceDescriptor(controller, request);
2028         }
2029 
2030         @Override
2031         public ListNamespaceDescriptorsResponse listNamespaceDescriptors(RpcController controller, ListNamespaceDescriptorsRequest request) throws ServiceException {
2032           return stub.listNamespaceDescriptors(controller, request);
2033         }
2034 
2035         @Override
2036         public ListTableDescriptorsByNamespaceResponse listTableDescriptorsByNamespace(RpcController controller, ListTableDescriptorsByNamespaceRequest request) throws ServiceException {
2037           return stub.listTableDescriptorsByNamespace(controller, request);
2038         }
2039 
2040         @Override
2041         public ListTableNamesByNamespaceResponse listTableNamesByNamespace(RpcController controller,
2042               ListTableNamesByNamespaceRequest request) throws ServiceException {
2043           return stub.listTableNamesByNamespace(controller, request);
2044         }
2045 
2046         @Override
2047         public void close() {
2048           release(this.mss);
2049         }
2050 
2051         @Override
2052         public GetSchemaAlterStatusResponse getSchemaAlterStatus(
2053             RpcController controller, GetSchemaAlterStatusRequest request)
2054             throws ServiceException {
2055           return stub.getSchemaAlterStatus(controller, request);
2056         }
2057 
2058         @Override
2059         public GetTableDescriptorsResponse getTableDescriptors(
2060             RpcController controller, GetTableDescriptorsRequest request)
2061             throws ServiceException {
2062           return stub.getTableDescriptors(controller, request);
2063         }
2064 
2065         @Override
2066         public GetTableNamesResponse getTableNames(
2067             RpcController controller, GetTableNamesRequest request)
2068             throws ServiceException {
2069           return stub.getTableNames(controller, request);
2070         }
2071 
2072         @Override
2073         public GetClusterStatusResponse getClusterStatus(
2074             RpcController controller, GetClusterStatusRequest request)
2075             throws ServiceException {
2076           return stub.getClusterStatus(controller, request);
2077         }
2078       };
2079     }
2080  
2081 
2082     private static void release(MasterServiceState mss) {
2083       if (mss != null && mss.connection != null) {
2084         ((HConnectionImplementation)mss.connection).releaseMaster(mss);
2085       }
2086     }
2087 
2088     private boolean isKeepAliveMasterConnectedAndRunning(MasterServiceState mss) {
2089       if (mss.getStub() == null){
2090         return false;
2091       }
2092       try {
2093         return mss.isMasterRunning();
2094       } catch (UndeclaredThrowableException e) {
2095         // It's somehow messy, but we can receive exceptions such as
2096         //  java.net.ConnectException but they're not declared. So we catch it...
2097         LOG.info("Master connection is not running anymore", e.getUndeclaredThrowable());
2098         return false;
2099       } catch (ServiceException se) {
2100         LOG.warn("Checking master connection", se);
2101         return false;
2102       }
2103     }
2104 
2105     void releaseMaster(MasterServiceState mss) {
2106       if (mss.getStub() == null) return;
2107       synchronized (masterAndZKLock) {
2108         --mss.userCount;
2109         if (mss.userCount <= 0) {
2110           mss.keepAliveUntil = System.currentTimeMillis() + keepAlive;
2111         }
2112       }
2113     }
2114 
2115     private void closeMasterService(MasterServiceState mss) {
2116       if (mss.getStub() != null) {
2117         LOG.info("Closing master protocol: " + mss);
2118         mss.clearStub();
2119       }
2120       mss.userCount = 0;
2121     }
2122 
2123     /**
2124      * Immediate close of the shared master. Can be by the delayed close or when closing the
2125      * connection itself.
2126      */
2127     private void closeMaster() {
2128       synchronized (masterAndZKLock) {
2129         closeMasterService(masterServiceState);
2130       }
2131     }
2132 
2133     void updateCachedLocation(HRegionInfo hri, HRegionLocation source,
2134                               ServerName serverName, long seqNum) {
2135       HRegionLocation newHrl = new HRegionLocation(hri, serverName, seqNum);
2136       cacheLocation(hri.getTable(), source, newHrl);
2137     }
2138 
2139    /**
2140     * Deletes the cached location of the region if necessary, based on some error from source.
2141     * @param hri The region in question.
2142     * @param source The source of the error that prompts us to invalidate cache.
2143     */
2144    void deleteCachedLocation(HRegionInfo hri, HRegionLocation source) {
2145      ConcurrentMap<byte[], HRegionLocation> tableLocations = getTableLocations(hri.getTable());
2146      tableLocations.remove(hri.getStartKey(), source);
2147    }
2148 
2149     @Override
2150     public void deleteCachedRegionLocation(final HRegionLocation location) {
2151       if (location == null) {
2152         return;
2153       }
2154 
2155       HRegionLocation removedLocation;
2156       TableName tableName = location.getRegionInfo().getTable();
2157       Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
2158       removedLocation = tableLocations.remove(location.getRegionInfo().getStartKey());
2159       if (LOG.isDebugEnabled() && removedLocation != null) {
2160         LOG.debug("Removed " +
2161             location.getRegionInfo().getRegionNameAsString() +
2162             " for tableName=" + tableName +
2163             " from cache");
2164       }
2165     }
2166 
2167     /**
2168      * Update the location with the new value (if the exception is a RegionMovedException)
2169      * or delete it from the cache. Does nothing if we can be sure from the exception that
2170      * the location is still accurate, or if the cache has already been updated.
2171      * @param exception an object (to simplify user code) on which we will try to find a nested
2172      *                  or wrapped or both RegionMovedException
2173      * @param source server that is the source of the location update.
2174      */
2175     @Override
2176     public void updateCachedLocations(final TableName tableName, byte[] rowkey,
2177       final Object exception, final HRegionLocation source) {
2178       if (rowkey == null || tableName == null) {
2179         LOG.warn("Coding error, see method javadoc. row=" + (rowkey == null ? "null" : rowkey) +
2180             ", tableName=" + (tableName == null ? "null" : tableName));
2181         return;
2182       }
2183 
2184       if (source == null || source.getServerName() == null){
2185         // This should not happen, but let's secure ourselves.
2186         return;
2187       }
2188 
2189       // Is it something we have already updated?
2190       final HRegionLocation oldLocation = getCachedLocation(tableName, rowkey);
2191       if (oldLocation == null || !source.getServerName().equals(oldLocation.getServerName())) {
2192         // There is no such location in the cache (it's been removed already) or
2193         // the cache has already been refreshed with a different location.  => nothing to do
2194         return;
2195       }
2196 
2197       HRegionInfo regionInfo = oldLocation.getRegionInfo();
2198       Throwable cause = findException(exception);
2199       if (cause != null) {
2200         if (cause instanceof RegionTooBusyException || cause instanceof RegionOpeningException) {
2201           // We know that the region is still on this region server
2202           return;
2203         }
2204 
2205         if (cause instanceof RegionMovedException) {
2206           RegionMovedException rme = (RegionMovedException) cause;
2207           if (LOG.isTraceEnabled()) {
2208             LOG.trace("Region " + regionInfo.getRegionNameAsString() + " moved to " +
2209                 rme.getHostname() + ":" + rme.getPort() +
2210                 " according to " + source.getHostnamePort());
2211           }
2212           // We know that the region is not anymore on this region server, but we know
2213           //  the new location.
2214           updateCachedLocation(
2215               regionInfo, source, rme.getServerName(), rme.getLocationSeqNum());
2216           return;
2217         }
2218       }
2219 
2220       // If we're here, it means that can cannot be sure about the location, so we remove it from
2221       //  the cache.
2222       deleteCachedLocation(regionInfo, source);
2223     }
2224 
2225     @Override
2226     public void updateCachedLocations(final byte[] tableName, byte[] rowkey,
2227       final Object exception, final HRegionLocation source) {
2228       updateCachedLocations(TableName.valueOf(tableName), rowkey, exception, source);
2229     }
2230 
2231     @Override
2232     @Deprecated
2233     public void processBatch(List<? extends Row> list,
2234         final TableName tableName,
2235         ExecutorService pool,
2236         Object[] results) throws IOException, InterruptedException {
2237       // This belongs in HTable!!! Not in here.  St.Ack
2238 
2239       // results must be the same size as list
2240       if (results.length != list.size()) {
2241         throw new IllegalArgumentException(
2242           "argument results must be the same size as argument list");
2243       }
2244       processBatchCallback(list, tableName, pool, results, null);
2245     }
2246 
2247     @Override
2248     @Deprecated
2249     public void processBatch(List<? extends Row> list,
2250         final byte[] tableName,
2251         ExecutorService pool,
2252         Object[] results) throws IOException, InterruptedException {
2253       processBatch(list, TableName.valueOf(tableName), pool, results);
2254     }
2255 
2256     /**
2257      * Send the queries in parallel on the different region servers. Retries on failures.
2258      * If the method returns it means that there is no error, and the 'results' array will
2259      * contain no exception. On error, an exception is thrown, and the 'results' array will
2260      * contain results and exceptions.
2261      * @deprecated since 0.96 - Use {@link HTable#processBatchCallback} instead
2262      */
2263     @Override
2264     @Deprecated
2265     public <R> void processBatchCallback(
2266       List<? extends Row> list,
2267       TableName tableName,
2268       ExecutorService pool,
2269       Object[] results,
2270       Batch.Callback<R> callback)
2271       throws IOException, InterruptedException {
2272 
2273       // To fulfill the original contract, we have a special callback. This callback
2274       //  will set the results in the Object array.
2275       ObjectResultFiller<R> cb = new ObjectResultFiller<R>(results, callback);
2276       AsyncProcess<?> asyncProcess = createAsyncProcess(tableName, pool, cb, conf);
2277 
2278       // We're doing a submit all. This way, the originalIndex will match the initial list.
2279       asyncProcess.submitAll(list);
2280       asyncProcess.waitUntilDone();
2281 
2282       if (asyncProcess.hasError()) {
2283         throw asyncProcess.getErrors();
2284       }
2285     }
2286 
2287     @Override
2288     @Deprecated
2289     public <R> void processBatchCallback(
2290       List<? extends Row> list,
2291       byte[] tableName,
2292       ExecutorService pool,
2293       Object[] results,
2294       Batch.Callback<R> callback)
2295       throws IOException, InterruptedException {
2296       processBatchCallback(list, TableName.valueOf(tableName), pool, results, callback);
2297     }
2298 
2299     // For tests.
2300     protected <R> AsyncProcess createAsyncProcess(TableName tableName, ExecutorService pool,
2301            AsyncProcess.AsyncProcessCallback<R> callback, Configuration conf) {
2302       return new AsyncProcess<R>(this, tableName, pool, callback, conf,
2303           RpcRetryingCallerFactory.instantiate(conf));
2304     }
2305 
2306 
2307     /**
2308      * Fill the result array for the interfaces using it.
2309      */
2310     private static class ObjectResultFiller<Res>
2311         implements AsyncProcess.AsyncProcessCallback<Res> {
2312 
2313       private final Object[] results;
2314       private Batch.Callback<Res> callback;
2315 
2316       ObjectResultFiller(Object[] results, Batch.Callback<Res> callback) {
2317         this.results = results;
2318         this.callback = callback;
2319       }
2320 
2321       @Override
2322       public void success(int pos, byte[] region, Row row, Res result) {
2323         assert pos < results.length;
2324         results[pos] = result;
2325         if (callback != null) {
2326           callback.update(region, row.getRow(), result);
2327         }
2328       }
2329 
2330       @Override
2331       public boolean failure(int pos, byte[] region, Row row, Throwable t) {
2332         assert pos < results.length;
2333         results[pos] = t;
2334         //Batch.Callback<Res> was not called on failure in 0.94. We keep this.
2335         return true; // we want to have this failure in the failures list.
2336       }
2337 
2338       @Override
2339       public boolean retriableFailure(int originalIndex, Row row, byte[] region,
2340                                       Throwable exception) {
2341         return true; // we retry
2342       }
2343     }
2344 
2345 
2346     /*
2347      * Return the number of cached region for a table. It will only be called
2348      * from a unit test.
2349      */
2350     int getNumberOfCachedRegionLocations(final TableName tableName) {
2351       Map<byte[], HRegionLocation> tableLocs = this.cachedRegionLocations.get(tableName);
2352       if (tableLocs == null) {
2353         return 0;
2354       }
2355       return tableLocs.values().size();
2356     }
2357 
2358     /**
2359      * Check the region cache to see whether a region is cached yet or not.
2360      * Called by unit tests.
2361      * @param tableName tableName
2362      * @param row row
2363      * @return Region cached or not.
2364      */
2365     boolean isRegionCached(TableName tableName, final byte[] row) {
2366       HRegionLocation location = getCachedLocation(tableName, row);
2367       return location != null;
2368     }
2369 
2370     @Override
2371     public void setRegionCachePrefetch(final TableName tableName,
2372         final boolean enable) {
2373       if (!enable) {
2374         regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName.getName()));
2375       }
2376       else {
2377         regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName.getName()));
2378       }
2379     }
2380 
2381     @Override
2382     public void setRegionCachePrefetch(final byte[] tableName,
2383         final boolean enable) {
2384       setRegionCachePrefetch(TableName.valueOf(tableName), enable);
2385     }
2386 
2387     @Override
2388     public boolean getRegionCachePrefetch(TableName tableName) {
2389       return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName.getName()));
2390     }
2391 
2392     @Override
2393     public boolean getRegionCachePrefetch(byte[] tableName) {
2394       return getRegionCachePrefetch(TableName.valueOf(tableName));
2395     }
2396 
2397     @Override
2398     public void abort(final String msg, Throwable t) {
2399       if (t instanceof KeeperException.SessionExpiredException
2400         && keepAliveZookeeper != null) {
2401         synchronized (masterAndZKLock) {
2402           if (keepAliveZookeeper != null) {
2403             LOG.warn("This client just lost it's session with ZooKeeper," +
2404               " closing it." +
2405               " It will be recreated next time someone needs it", t);
2406             closeZooKeeperWatcher();
2407           }
2408         }
2409       } else {
2410         if (t != null) {
2411           LOG.fatal(msg, t);
2412         } else {
2413           LOG.fatal(msg);
2414         }
2415         this.aborted = true;
2416         close();
2417         this.closed = true;
2418       }
2419     }
2420 
2421     @Override
2422     public boolean isClosed() {
2423       return this.closed;
2424     }
2425 
2426     @Override
2427     public boolean isAborted(){
2428       return this.aborted;
2429     }
2430 
2431     @Override
2432     public int getCurrentNrHRS() throws IOException {
2433       return this.registry.getCurrentNrHRS();
2434     }
2435 
2436     /**
2437      * Increment this client's reference count.
2438      */
2439     void incCount() {
2440       ++refCount;
2441     }
2442 
2443     /**
2444      * Decrement this client's reference count.
2445      */
2446     void decCount() {
2447       if (refCount > 0) {
2448         --refCount;
2449       }
2450     }
2451 
2452     /**
2453      * Return if this client has no reference
2454      *
2455      * @return true if this client has no reference; false otherwise
2456      */
2457     boolean isZeroReference() {
2458       return refCount == 0;
2459     }
2460 
2461     void internalClose() {
2462       if (this.closed) {
2463         return;
2464       }
2465       delayedClosing.stop("Closing connection");
2466       closeMaster();
2467       shutdownBatchPool();
2468       this.closed = true;
2469       closeZooKeeperWatcher();
2470       this.stubs.clear();
2471       if (clusterStatusListener != null) {
2472         clusterStatusListener.close();
2473       }
2474     }
2475 
2476     @Override
2477     public void close() {
2478       if (managed) {
2479         if (aborted) {
2480           HConnectionManager.deleteStaleConnection(this);
2481         } else {
2482           HConnectionManager.deleteConnection(this, false);
2483         }
2484       } else {
2485         internalClose();
2486       }
2487     }
2488 
2489     /**
2490      * Close the connection for good, regardless of what the current value of
2491      * {@link #refCount} is. Ideally, {@link #refCount} should be zero at this
2492      * point, which would be the case if all of its consumers close the
2493      * connection. However, on the off chance that someone is unable to close
2494      * the connection, perhaps because it bailed out prematurely, the method
2495      * below will ensure that this {@link HConnection} instance is cleaned up.
2496      * Caveat: The JVM may take an unknown amount of time to call finalize on an
2497      * unreachable object, so our hope is that every consumer cleans up after
2498      * itself, like any good citizen.
2499      */
2500     @Override
2501     protected void finalize() throws Throwable {
2502       super.finalize();
2503       // Pretend as if we are about to release the last remaining reference
2504       refCount = 1;
2505       close();
2506     }
2507 
2508     @Override
2509     public HTableDescriptor[] listTables() throws IOException {
2510       MasterKeepAliveConnection master = getKeepAliveMasterService();
2511       try {
2512         GetTableDescriptorsRequest req =
2513           RequestConverter.buildGetTableDescriptorsRequest((List<TableName>)null);
2514         return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2515       } catch (ServiceException se) {
2516         throw ProtobufUtil.getRemoteException(se);
2517       } finally {
2518         master.close();
2519       }
2520     }
2521 
2522     @Override
2523     public String[] getTableNames() throws IOException {
2524       TableName[] tableNames = listTableNames();
2525       String result[] = new String[tableNames.length];
2526       for (int i = 0; i < tableNames.length; i++) {
2527         result[i] = tableNames[i].getNameAsString();
2528       }
2529       return result;
2530     }
2531 
2532     @Override
2533     public TableName[] listTableNames() throws IOException {
2534       MasterKeepAliveConnection master = getKeepAliveMasterService();
2535       try {
2536         return ProtobufUtil.getTableNameArray(master.getTableNames(null,
2537             GetTableNamesRequest.newBuilder().build())
2538           .getTableNamesList());
2539       } catch (ServiceException se) {
2540         throw ProtobufUtil.getRemoteException(se);
2541       } finally {
2542         master.close();
2543       }
2544     }
2545 
2546     @Override
2547     public HTableDescriptor[] getHTableDescriptorsByTableName(
2548         List<TableName> tableNames) throws IOException {
2549       if (tableNames == null || tableNames.isEmpty()) return new HTableDescriptor[0];
2550       MasterKeepAliveConnection master = getKeepAliveMasterService();
2551       try {
2552         GetTableDescriptorsRequest req =
2553           RequestConverter.buildGetTableDescriptorsRequest(tableNames);
2554         return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2555       } catch (ServiceException se) {
2556         throw ProtobufUtil.getRemoteException(se);
2557       } finally {
2558         master.close();
2559       }
2560     }
2561 
2562     @Override
2563     public HTableDescriptor[] getHTableDescriptors(
2564         List<String> names) throws IOException {
2565       List<TableName> tableNames = new ArrayList(names.size());
2566       for(String name : names) {
2567         tableNames.add(TableName.valueOf(name));
2568       }
2569 
2570       return getHTableDescriptorsByTableName(tableNames);
2571     }
2572 
2573     /**
2574      * Connects to the master to get the table descriptor.
2575      * @param tableName table name
2576      * @return
2577      * @throws IOException if the connection to master fails or if the table
2578      *  is not found.
2579      */
2580     @Override
2581     public HTableDescriptor getHTableDescriptor(final TableName tableName)
2582     throws IOException {
2583       if (tableName == null) return null;
2584       if (tableName.equals(TableName.META_TABLE_NAME)) {
2585         return HTableDescriptor.META_TABLEDESC;
2586       }
2587       MasterKeepAliveConnection master = getKeepAliveMasterService();
2588       GetTableDescriptorsResponse htds;
2589       try {
2590         GetTableDescriptorsRequest req =
2591           RequestConverter.buildGetTableDescriptorsRequest(tableName);
2592         htds = master.getTableDescriptors(null, req);
2593       } catch (ServiceException se) {
2594         throw ProtobufUtil.getRemoteException(se);
2595       } finally {
2596         master.close();
2597       }
2598       if (!htds.getTableSchemaList().isEmpty()) {
2599         return HTableDescriptor.convert(htds.getTableSchemaList().get(0));
2600       }
2601       throw new TableNotFoundException(tableName.getNameAsString());
2602     }
2603 
2604     @Override
2605     public HTableDescriptor getHTableDescriptor(final byte[] tableName)
2606     throws IOException {
2607       return getHTableDescriptor(TableName.valueOf(tableName));
2608     }
2609   }
2610 
2611   /**
2612    * The record of errors for servers.
2613    */
2614   static class ServerErrorTracker {
2615     // We need a concurrent map here, as we could have multiple threads updating it in parallel.
2616     private final ConcurrentMap<HRegionLocation, ServerErrors> errorsByServer =
2617         new ConcurrentHashMap<HRegionLocation, ServerErrors>();
2618     private final long canRetryUntil;
2619     private final int maxRetries;
2620     private final String startTrackingTime;
2621 
2622     public ServerErrorTracker(long timeout, int maxRetries) {
2623       this.maxRetries = maxRetries;
2624       this.canRetryUntil = EnvironmentEdgeManager.currentTimeMillis() + timeout;
2625       this.startTrackingTime = new Date().toString();
2626     }
2627 
2628     /**
2629      * We stop to retry when we have exhausted BOTH the number of retries and the time allocated.
2630      */
2631     boolean canRetryMore(int numRetry) {
2632       // If there is a single try we must not take into account the time.
2633       return numRetry < maxRetries || (maxRetries > 1 &&
2634           EnvironmentEdgeManager.currentTimeMillis() < this.canRetryUntil);
2635     }
2636 
2637     /**
2638      * Calculates the back-off time for a retrying request to a particular server.
2639      *
2640      * @param server    The server in question.
2641      * @param basePause The default hci pause.
2642      * @return The time to wait before sending next request.
2643      */
2644     long calculateBackoffTime(HRegionLocation server, long basePause) {
2645       long result;
2646       ServerErrors errorStats = errorsByServer.get(server);
2647       if (errorStats != null) {
2648         result = ConnectionUtils.getPauseTime(basePause, errorStats.retries.get());
2649       } else {
2650         result = 0; // yes, if the server is not in our list we don't wait before retrying.
2651       }
2652       return result;
2653     }
2654 
2655     /**
2656      * Reports that there was an error on the server to do whatever bean-counting necessary.
2657      *
2658      * @param server The server in question.
2659      */
2660     void reportServerError(HRegionLocation server) {
2661       ServerErrors errors = errorsByServer.get(server);
2662       if (errors != null) {
2663         errors.addError();
2664       } else {
2665         errors = errorsByServer.putIfAbsent(server, new ServerErrors());
2666         if (errors != null){
2667           errors.addError();
2668         }
2669       }
2670     }
2671 
2672     String getStartTrackingTime() {
2673       return startTrackingTime;
2674     }
2675 
2676     /**
2677      * The record of errors for a server.
2678      */
2679     private static class ServerErrors {
2680       public final AtomicInteger retries = new AtomicInteger(0);
2681 
2682       public void addError() {
2683         retries.incrementAndGet();
2684       }
2685     }
2686   }
2687 
2688   /**
2689    * Look for an exception we know in the remote exception:
2690    * - hadoop.ipc wrapped exceptions
2691    * - nested exceptions
2692    * 
2693    * Looks for: RegionMovedException / RegionOpeningException / RegionTooBusyException
2694    * @return null if we didn't find the exception, the exception otherwise.
2695    */
2696   public static Throwable findException(Object exception) {
2697     if (exception == null || !(exception instanceof Throwable)) {
2698       return null;
2699     }
2700     Throwable cur = (Throwable) exception;
2701     while (cur != null) {
2702       if (cur instanceof RegionMovedException || cur instanceof RegionOpeningException
2703           || cur instanceof RegionTooBusyException) {
2704         return cur;
2705       }
2706       if (cur instanceof RemoteException) {
2707         RemoteException re = (RemoteException) cur;
2708         cur = re.unwrapRemoteException(
2709             RegionOpeningException.class, RegionMovedException.class,
2710             RegionTooBusyException.class);
2711         if (cur == null) {
2712           cur = re.unwrapRemoteException();
2713         }
2714         // unwrapRemoteException can return the exception given as a parameter when it cannot
2715         //  unwrap it. In this case, there is no need to look further
2716         // noinspection ObjectEquality
2717         if (cur == re) {
2718           return null;
2719         }
2720       } else {
2721         cur = cur.getCause();
2722       }
2723     }
2724 
2725     return null;
2726   }
2727 
2728   /**
2729    * Set the number of retries to use serverside when trying to communicate
2730    * with another server over {@link HConnection}.  Used updating catalog
2731    * tables, etc.  Call this method before we create any Connections.
2732    * @param c The Configuration instance to set the retries into.
2733    * @param log Used to log what we set in here.
2734    */
2735   public static void setServerSideHConnectionRetries(final Configuration c, final String sn,
2736       final Log log) {
2737     int hcRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2738       HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2739     // Go big.  Multiply by 10.  If we can't get to meta after this many retries
2740     // then something seriously wrong.
2741     int serversideMultiplier = c.getInt("hbase.client.serverside.retries.multiplier", 10);
2742     int retries = hcRetries * serversideMultiplier;
2743     c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, retries);
2744     log.debug(sn + " HConnection server-to-server retries=" + retries);
2745   }
2746 }