View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.lang.reflect.Constructor;
24  import java.lang.reflect.UndeclaredThrowableException;
25  import java.net.SocketException;
26  import java.util.ArrayList;
27  import java.util.Date;
28  import java.util.HashSet;
29  import java.util.LinkedHashMap;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.Map.Entry;
33  import java.util.NavigableMap;
34  import java.util.Set;
35  import java.util.concurrent.ConcurrentHashMap;
36  import java.util.concurrent.ConcurrentMap;
37  import java.util.concurrent.ConcurrentSkipListMap;
38  import java.util.concurrent.ConcurrentSkipListSet;
39  import java.util.concurrent.CopyOnWriteArraySet;
40  import java.util.concurrent.ExecutorService;
41  import java.util.concurrent.LinkedBlockingQueue;
42  import java.util.concurrent.ThreadPoolExecutor;
43  import java.util.concurrent.TimeUnit;
44  import java.util.concurrent.atomic.AtomicBoolean;
45  import java.util.concurrent.atomic.AtomicInteger;
46  
47  import org.apache.commons.logging.Log;
48  import org.apache.commons.logging.LogFactory;
49  import org.apache.hadoop.classification.InterfaceAudience;
50  import org.apache.hadoop.classification.InterfaceStability;
51  import org.apache.hadoop.conf.Configuration;
52  import org.apache.hadoop.hbase.Chore;
53  import org.apache.hadoop.hbase.HBaseConfiguration;
54  import org.apache.hadoop.hbase.HConstants;
55  import org.apache.hadoop.hbase.HRegionInfo;
56  import org.apache.hadoop.hbase.HRegionLocation;
57  import org.apache.hadoop.hbase.HTableDescriptor;
58  import org.apache.hadoop.hbase.MasterNotRunningException;
59  import org.apache.hadoop.hbase.RegionTooBusyException;
60  import org.apache.hadoop.hbase.ServerName;
61  import org.apache.hadoop.hbase.Stoppable;
62  import org.apache.hadoop.hbase.TableName;
63  import org.apache.hadoop.hbase.TableNotEnabledException;
64  import org.apache.hadoop.hbase.TableNotFoundException;
65  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
66  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
67  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
68  import org.apache.hadoop.hbase.client.coprocessor.Batch;
69  import org.apache.hadoop.hbase.exceptions.RegionMovedException;
70  import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
71  import org.apache.hadoop.hbase.ipc.RpcClient;
72  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73  import org.apache.hadoop.hbase.protobuf.RequestConverter;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
75  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
76  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
77  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
78  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnResponse;
79  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionResponse;
80  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableResponse;
82  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnResponse;
83  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableResponse;
85  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse;
86  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse;
87  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
88  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse;
89  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsResponse;
90  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
91  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
92  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest;
93  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse;
94  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse;
96  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
97  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
98  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceResponse;
99  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnResponse;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableResponse;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionResponse;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionResponse;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterResponse;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionResponse;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.*;
115 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
116 import org.apache.hadoop.hbase.security.User;
117 import org.apache.hadoop.hbase.security.UserProvider;
118 import org.apache.hadoop.hbase.util.Bytes;
119 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
120 import org.apache.hadoop.hbase.util.Threads;
121 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
122 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
123 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
124 import org.apache.hadoop.ipc.RemoteException;
125 import org.apache.zookeeper.KeeperException;
126 
127 import com.google.protobuf.BlockingRpcChannel;
128 import com.google.protobuf.RpcController;
129 import com.google.protobuf.ServiceException;
130 
131 /**
132  * A non-instantiable class that manages creation of {@link HConnection}s.
133  * <p>The simplest way to use this class is by using {@link #createConnection(Configuration)}.
134  * This creates a new {@link HConnection} to the cluster that is managed by the caller.
135  * From this {@link HConnection} {@link HTableInterface} implementations are retrieved
136  * with {@link HConnection#getTable(byte[])}. Example:
137  * <pre>
138  * {@code
139  * HConnection connection = HConnectionManager.createConnection(config);
140  * HTableInterface table = connection.getTable("table1");
141  * try {
142  *   // Use the table as needed, for a single operation and a single thread
143  * } finally {
144  *   table.close();
145  *   connection.close();
146  * }} </pre>
147  * <p>The following logic and API will be removed in the future:
148  * <p>This class has a static Map of {@link HConnection} instances keyed by
149  * {@link Configuration}; all invocations of {@link #getConnection(Configuration)}
150  * that pass the same {@link Configuration} instance will be returned the same
151  * {@link  HConnection} instance (Adding properties to a Configuration
152  * instance does not change its object identity; for more on how this is done see
153  * {@link HConnectionKey}).  Sharing {@link HConnection}
154  * instances is usually what you want; all clients of the {@link HConnection}
155  * instances share the HConnections' cache of Region locations rather than each
156  * having to discover for itself the location of meta, etc.  It makes
157  * sense for the likes of the pool of HTables class {@link HTablePool}, for
158  * instance (If concerned that a single {@link HConnection} is insufficient
159  * for sharing amongst clients in say an heavily-multithreaded environment,
160  * in practise its not proven to be an issue.  Besides, {@link HConnection} is
161  * implemented atop Hadoop RPC and as of this writing, Hadoop RPC does a
162  * connection per cluster-member, exclusively).
163  *
164  * <p>But sharing connections makes clean up of {@link HConnection} instances a little awkward.
165  * Currently, clients cleanup by calling {@link #deleteConnection(Configuration)}. This will
166  * shutdown the zookeeper connection the HConnection was using and clean up all
167  * HConnection resources as well as stopping proxies to servers out on the
168  * cluster. Not running the cleanup will not end the world; it'll
169  * just stall the closeup some and spew some zookeeper connection failed
170  * messages into the log.  Running the cleanup on a {@link HConnection} that is
171  * subsequently used by another will cause breakage so be careful running
172  * cleanup.
173  * <p>To create a {@link HConnection} that is not shared by others, you can
174  * create a new {@link Configuration} instance, pass this new instance to
175  * {@link #getConnection(Configuration)}, and then when done, close it up by
176  * doing something like the following:
177  * <pre>
178  * {@code
179  * Configuration newConfig = new Configuration(originalConf);
180  * HConnection connection = HConnectionManager.getConnection(newConfig);
181  * // Use the connection to your hearts' delight and then when done...
182  * HConnectionManager.deleteConnection(newConfig, true);
183  * }
184  * </pre>
185  * <p>Cleanup used to be done inside in a shutdown hook.  On startup we'd
186  * register a shutdown hook that called {@link #deleteAllConnections()}
187  * on its way out but the order in which shutdown hooks run is not defined so
188  * were problematic for clients of HConnection that wanted to register their
189  * own shutdown hooks so we removed ours though this shifts the onus for
190  * cleanup to the client.
191  */
192 @SuppressWarnings("serial")
193 @InterfaceAudience.Public
194 @InterfaceStability.Evolving
195 public class HConnectionManager {
196   static final Log LOG = LogFactory.getLog(HConnectionManager.class);
197 
198   public static final String RETRIES_BY_SERVER_KEY = "hbase.client.retries.by.server";
199 
200   // An LRU Map of HConnectionKey -> HConnection (TableServer).  All
201   // access must be synchronized.  This map is not private because tests
202   // need to be able to tinker with it.
203   static final Map<HConnectionKey, HConnectionImplementation> CONNECTION_INSTANCES;
204 
205   public static final int MAX_CACHED_CONNECTION_INSTANCES;
206 
207   static {
208     // We set instances to one more than the value specified for {@link
209     // HConstants#ZOOKEEPER_MAX_CLIENT_CNXNS}. By default, the zk default max
210     // connections to the ensemble from the one client is 30, so in that case we
211     // should run into zk issues before the LRU hit this value of 31.
212     MAX_CACHED_CONNECTION_INSTANCES = HBaseConfiguration.create().getInt(
213       HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1;
214     CONNECTION_INSTANCES = new LinkedHashMap<HConnectionKey, HConnectionImplementation>(
215         (int) (MAX_CACHED_CONNECTION_INSTANCES / 0.75F) + 1, 0.75F, true) {
216       @Override
217       protected boolean removeEldestEntry(
218           Map.Entry<HConnectionKey, HConnectionImplementation> eldest) {
219          return size() > MAX_CACHED_CONNECTION_INSTANCES;
220        }
221     };
222   }
223 
224   /*
225    * Non-instantiable.
226    */
227   private HConnectionManager() {
228     super();
229   }
230 
231   /**
232    * Get the connection that goes with the passed <code>conf</code> configuration instance.
233    * If no current connection exists, method creates a new connection and keys it using
234    * connection-specific properties from the passed {@link Configuration}; see
235    * {@link HConnectionKey}.
236    * @param conf configuration
237    * @return HConnection object for <code>conf</code>
238    * @throws ZooKeeperConnectionException
239    */
240   @Deprecated
241   public static HConnection getConnection(final Configuration conf)
242   throws IOException {
243     HConnectionKey connectionKey = new HConnectionKey(conf);
244     synchronized (CONNECTION_INSTANCES) {
245       HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);
246       if (connection == null) {
247         connection = (HConnectionImplementation)createConnection(conf, true);
248         CONNECTION_INSTANCES.put(connectionKey, connection);
249       } else if (connection.isClosed()) {
250         HConnectionManager.deleteConnection(connectionKey, true);
251         connection = (HConnectionImplementation)createConnection(conf, true);
252         CONNECTION_INSTANCES.put(connectionKey, connection);
253       }
254       connection.incCount();
255       return connection;
256     }
257   }
258 
259   /**
260    * Create a new HConnection instance using the passed <code>conf</code> instance.
261    * <p>Note: This bypasses the usual HConnection life cycle management done by
262    * {@link #getConnection(Configuration)}. The caller is responsible for
263    * calling {@link HConnection#close()} on the returned connection instance.
264    *
265    * This is the recommended way to create HConnections.
266    * {@code
267    * HConnection connection = HConnectionManager.createConnection(conf);
268    * HTableInterface table = connection.getTable("mytable");
269    * table.get(...);
270    * ...
271    * table.close();
272    * connection.close();
273    * }
274    *
275    * @param conf configuration
276    * @return HConnection object for <code>conf</code>
277    * @throws ZooKeeperConnectionException
278    */
279   public static HConnection createConnection(Configuration conf)
280   throws IOException {
281     UserProvider provider = UserProvider.instantiate(conf);
282     return createConnection(conf, false, null, provider.getCurrent());
283   }
284 
285   /**
286    * Create a new HConnection instance using the passed <code>conf</code> instance.
287    * <p>Note: This bypasses the usual HConnection life cycle management done by
288    * {@link #getConnection(Configuration)}. The caller is responsible for
289    * calling {@link HConnection#close()} on the returned connection instance.
290    * This is the recommended way to create HConnections.
291    * {@code
292    * ExecutorService pool = ...;
293    * HConnection connection = HConnectionManager.createConnection(conf, pool);
294    * HTableInterface table = connection.getTable("mytable");
295    * table.get(...);
296    * ...
297    * table.close();
298    * connection.close();
299    * }
300    * @param conf configuration
301    * @param pool the thread pool to use for batch operation in HTables used via this HConnection
302    * @return HConnection object for <code>conf</code>
303    * @throws ZooKeeperConnectionException
304    */
305   public static HConnection createConnection(Configuration conf, ExecutorService pool)
306   throws IOException {
307     UserProvider provider = UserProvider.instantiate(conf);
308     return createConnection(conf, false, pool, provider.getCurrent());
309   }
310 
311   /**
312    * Create a new HConnection instance using the passed <code>conf</code> instance.
313    * <p>Note: This bypasses the usual HConnection life cycle management done by
314    * {@link #getConnection(Configuration)}. The caller is responsible for
315    * calling {@link HConnection#close()} on the returned connection instance.
316    * This is the recommended way to create HConnections.
317    * {@code
318    * ExecutorService pool = ...;
319    * HConnection connection = HConnectionManager.createConnection(conf, pool);
320    * HTableInterface table = connection.getTable("mytable");
321    * table.get(...);
322    * ...
323    * table.close();
324    * connection.close();
325    * }
326    * @param conf configuration
327    * @param user the user the connection is for
328    * @return HConnection object for <code>conf</code>
329    * @throws ZooKeeperConnectionException
330    */
331   public static HConnection createConnection(Configuration conf, User user)
332   throws IOException {
333     return createConnection(conf, false, null, user);
334   }
335 
336   /**
337    * Create a new HConnection instance using the passed <code>conf</code> instance.
338    * <p>Note: This bypasses the usual HConnection life cycle management done by
339    * {@link #getConnection(Configuration)}. The caller is responsible for
340    * calling {@link HConnection#close()} on the returned connection instance.
341    * This is the recommended way to create HConnections.
342    * {@code
343    * ExecutorService pool = ...;
344    * HConnection connection = HConnectionManager.createConnection(conf, pool);
345    * HTableInterface table = connection.getTable("mytable");
346    * table.get(...);
347    * ...
348    * table.close();
349    * connection.close();
350    * }
351    * @param conf configuration
352    * @param pool the thread pool to use for batch operation in HTables used via this HConnection
353    * @param user the user the connection is for
354    * @return HConnection object for <code>conf</code>
355    * @throws ZooKeeperConnectionException
356    */
357   public static HConnection createConnection(Configuration conf, ExecutorService pool, User user)
358   throws IOException {
359     return createConnection(conf, false, pool, user);
360   }
361 
362   @Deprecated
363   static HConnection createConnection(final Configuration conf, final boolean managed)
364       throws IOException {
365     UserProvider provider = UserProvider.instantiate(conf);
366     return createConnection(conf, managed, null, provider.getCurrent());
367   }
368 
369   @Deprecated
370   static HConnection createConnection(final Configuration conf, final boolean managed,
371       final ExecutorService pool, final User user)
372   throws IOException {
373     String className = conf.get("hbase.client.connection.impl",
374       HConnectionManager.HConnectionImplementation.class.getName());
375     Class<?> clazz = null;
376     try {
377       clazz = Class.forName(className);
378     } catch (ClassNotFoundException e) {
379       throw new IOException(e);
380     }
381     try {
382       // Default HCM#HCI is not accessible; make it so before invoking.
383       Constructor<?> constructor =
384         clazz.getDeclaredConstructor(Configuration.class,
385           boolean.class, ExecutorService.class, User.class);
386       constructor.setAccessible(true);
387       return (HConnection) constructor.newInstance(conf, managed, pool, user);
388     } catch (Exception e) {
389       throw new IOException(e);
390     }
391   }
392 
393   /**
394    * Delete connection information for the instance specified by passed configuration.
395    * If there are no more references to the designated connection connection, this method will
396    * then close connection to the zookeeper ensemble and let go of all associated resources.
397    *
398    * @param conf configuration whose identity is used to find {@link HConnection} instance.
399    * @deprecated
400    */
401   public static void deleteConnection(Configuration conf) {
402     deleteConnection(new HConnectionKey(conf), false);
403   }
404 
405   /**
406    * Cleanup a known stale connection.
407    * This will then close connection to the zookeeper ensemble and let go of all resources.
408    *
409    * @param connection
410    * @deprecated
411    */
412   public static void deleteStaleConnection(HConnection connection) {
413     deleteConnection(connection, true);
414   }
415 
416   /**
417    * Delete information for all connections. Close or not the connection, depending on the
418    *  staleConnection boolean and the ref count. By default, you should use it with
419    *  staleConnection to true.
420    * @deprecated
421    */
422   public static void deleteAllConnections(boolean staleConnection) {
423     synchronized (CONNECTION_INSTANCES) {
424       Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
425       connectionKeys.addAll(CONNECTION_INSTANCES.keySet());
426       for (HConnectionKey connectionKey : connectionKeys) {
427         deleteConnection(connectionKey, staleConnection);
428       }
429       CONNECTION_INSTANCES.clear();
430     }
431   }
432 
433   /**
434    * Delete information for all connections..
435    * @deprecated kept for backward compatibility, but the behavior is broken. HBASE-8983
436    */
437   @Deprecated
438   public static void deleteAllConnections() {
439     deleteAllConnections(false);
440   }
441 
442 
443   @Deprecated
444   private static void deleteConnection(HConnection connection, boolean staleConnection) {
445     synchronized (CONNECTION_INSTANCES) {
446       for (Entry<HConnectionKey, HConnectionImplementation> e: CONNECTION_INSTANCES.entrySet()) {
447         if (e.getValue() == connection) {
448           deleteConnection(e.getKey(), staleConnection);
449           break;
450         }
451       }
452     }
453   }
454 
455   @Deprecated
456   private static void deleteConnection(HConnectionKey connectionKey, boolean staleConnection) {
457     synchronized (CONNECTION_INSTANCES) {
458       HConnectionImplementation connection = CONNECTION_INSTANCES.get(connectionKey);
459       if (connection != null) {
460         connection.decCount();
461         if (connection.isZeroReference() || staleConnection) {
462           CONNECTION_INSTANCES.remove(connectionKey);
463           connection.internalClose();
464         }
465       } else {
466         LOG.error("Connection not found in the list, can't delete it "+
467           "(connection key=" + connectionKey + "). May be the key was modified?", new Exception());
468       }
469     }
470   }
471 
472   /**
473    * It is provided for unit test cases which verify the behavior of region
474    * location cache prefetch.
475    * @return Number of cached regions for the table.
476    * @throws ZooKeeperConnectionException
477    */
478   static int getCachedRegionCount(Configuration conf, final TableName tableName)
479   throws IOException {
480     return execute(new HConnectable<Integer>(conf) {
481       @Override
482       public Integer connect(HConnection connection) {
483         return ((HConnectionImplementation)connection).getNumberOfCachedRegionLocations(tableName);
484       }
485     });
486   }
487 
488   /**
489    * It's provided for unit test cases which verify the behavior of region
490    * location cache prefetch.
491    * @return true if the region where the table and row reside is cached.
492    * @throws ZooKeeperConnectionException
493    */
494   static boolean isRegionCached(Configuration conf,
495                                 final TableName tableName,
496                                 final byte[] row)
497   throws IOException {
498     return execute(new HConnectable<Boolean>(conf) {
499       @Override
500       public Boolean connect(HConnection connection) {
501         return ((HConnectionImplementation) connection).isRegionCached(tableName, row);
502       }
503     });
504   }
505 
506   /**
507    * This convenience method invokes the given {@link HConnectable#connect}
508    * implementation using a {@link HConnection} instance that lasts just for the
509    * duration of the invocation.
510    *
511    * @param <T> the return type of the connect method
512    * @param connectable the {@link HConnectable} instance
513    * @return the value returned by the connect method
514    * @throws IOException
515    */
516   @InterfaceAudience.Private
517   public static <T> T execute(HConnectable<T> connectable) throws IOException {
518     if (connectable == null || connectable.conf == null) {
519       return null;
520     }
521     Configuration conf = connectable.conf;
522     HConnection connection = HConnectionManager.getConnection(conf);
523     boolean connectSucceeded = false;
524     try {
525       T returnValue = connectable.connect(connection);
526       connectSucceeded = true;
527       return returnValue;
528     } finally {
529       try {
530         connection.close();
531       } catch (Exception e) {
532         if (connectSucceeded) {
533           throw new IOException("The connection to " + connection
534               + " could not be deleted.", e);
535         }
536       }
537     }
538   }
539 
540   /** Encapsulates connection to zookeeper and regionservers.*/
541   @edu.umd.cs.findbugs.annotations.SuppressWarnings(
542       value="AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION",
543       justification="Access to the conncurrent hash map is under a lock so should be fine.")
544   static class HConnectionImplementation implements HConnection, Closeable {
545     static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);
546     private final long pause;
547     private final int numTries;
548     final int rpcTimeout;
549     private final int prefetchRegionLimit;
550 
551     private volatile boolean closed;
552     private volatile boolean aborted;
553 
554     // package protected for the tests
555     ClusterStatusListener clusterStatusListener;
556 
557     private final Object userRegionLock = new Object();
558 
559     // We have a single lock for master & zk to prevent deadlocks. Having
560     //  one lock for ZK and one lock for master is not possible:
561     //  When creating a connection to master, we need a connection to ZK to get
562     //  its address. But another thread could have taken the ZK lock, and could
563     //  be waiting for the master lock => deadlock.
564     private final Object masterAndZKLock = new Object();
565 
566     private long keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
567     private final DelayedClosing delayedClosing =
568       DelayedClosing.createAndStart(this);
569 
570     // thread executor shared by all HTableInterface instances created
571     // by this connection
572     private volatile ExecutorService batchPool = null;
573     private volatile boolean cleanupPool = false;
574 
575     private final Configuration conf;
576 
577     // Client rpc instance.
578     private RpcClient rpcClient;
579 
580     /**
581       * Map of table to table {@link HRegionLocation}s.
582       */
583     private final ConcurrentMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>
584         cachedRegionLocations =
585       new ConcurrentHashMap<TableName, ConcurrentSkipListMap<byte[], HRegionLocation>>();
586 
587     // The presence of a server in the map implies it's likely that there is an
588     // entry in cachedRegionLocations that map to this server; but the absence
589     // of a server in this map guarentees that there is no entry in cache that
590     // maps to the absent server.
591     // The access to this attribute must be protected by a lock on cachedRegionLocations
592     private final Set<ServerName> cachedServers = new ConcurrentSkipListSet<ServerName>();
593 
594     // region cache prefetch is enabled by default. this set contains all
595     // tables whose region cache prefetch are disabled.
596     private final Set<Integer> regionCachePrefetchDisabledTables =
597       new CopyOnWriteArraySet<Integer>();
598 
599     private int refCount;
600 
601     // indicates whether this connection's life cycle is managed (by us)
602     private boolean managed;
603 
604     private User user;
605 
606     /**
607      * Cluster registry of basic info such as clusterid and meta region location.
608      */
609      Registry registry;
610 
611      HConnectionImplementation(Configuration conf, boolean managed) throws IOException {
612        this(conf, managed, null, null);
613      }
614 
615     /**
616      * constructor
617      * @param conf Configuration object
618      * @param managed If true, does not do full shutdown on close; i.e. cleanup of connection
619      * to zk and shutdown of all services; we just close down the resources this connection was
620      * responsible for and decrement usage counters.  It is up to the caller to do the full
621      * cleanup.  It is set when we want have connection sharing going on -- reuse of zk connection,
622      * and cached region locations, established regionserver connections, etc.  When connections
623      * are shared, we have reference counting going on and will only do full cleanup when no more
624      * users of an HConnectionImplementation instance.
625      */
626     HConnectionImplementation(Configuration conf, boolean managed,
627         ExecutorService pool, User user) throws IOException {
628       this(conf);
629       this.user = user;
630       this.batchPool = pool;
631       this.managed = managed;
632       this.registry = setupRegistry();
633       retrieveClusterId();
634 
635       this.rpcClient = new RpcClient(this.conf, this.clusterId);
636 
637       // Do we publish the status?
638       boolean shouldListen = conf.getBoolean(HConstants.STATUS_PUBLISHED,
639           HConstants.STATUS_PUBLISHED_DEFAULT);
640       Class<? extends ClusterStatusListener.Listener> listenerClass =
641           conf.getClass(ClusterStatusListener.STATUS_LISTENER_CLASS,
642               ClusterStatusListener.DEFAULT_STATUS_LISTENER_CLASS,
643               ClusterStatusListener.Listener.class);
644       if (shouldListen) {
645         if (listenerClass == null) {
646           LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
647               ClusterStatusListener.STATUS_LISTENER_CLASS + " is not set - not listening status");
648         } else {
649           clusterStatusListener = new ClusterStatusListener(
650               new ClusterStatusListener.DeadServerHandler() {
651                 @Override
652                 public void newDead(ServerName sn) {
653                   clearCaches(sn);
654                   rpcClient.cancelConnections(sn.getHostname(), sn.getPort(),
655                       new SocketException(sn.getServerName() +
656                           " is dead: closing its connection."));
657                 }
658               }, conf, listenerClass);
659         }
660       }
661     }
662 
663 
664     /**
665      * For tests.
666      */
667     protected HConnectionImplementation(Configuration conf) {
668       this.conf = conf;
669       this.closed = false;
670       this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
671           HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
672       this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
673           HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
674       this.rpcTimeout = conf.getInt(
675           HConstants.HBASE_RPC_TIMEOUT_KEY,
676           HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
677       this.prefetchRegionLimit = conf.getInt(
678           HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
679           HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
680     }
681 
682     @Override
683     public HTableInterface getTable(String tableName) throws IOException {
684       return getTable(TableName.valueOf(tableName));
685     }
686 
687     @Override
688     public HTableInterface getTable(byte[] tableName) throws IOException {
689       return getTable(TableName.valueOf(tableName));
690     }
691 
692     @Override
693     public HTableInterface getTable(TableName tableName) throws IOException {
694       return getTable(tableName, getBatchPool());
695     }
696 
697     @Override
698     public HTableInterface getTable(String tableName, ExecutorService pool) throws IOException {
699       return getTable(TableName.valueOf(tableName), pool);
700     }
701 
702     @Override
703     public HTableInterface getTable(byte[] tableName, ExecutorService pool) throws IOException {
704       return getTable(TableName.valueOf(tableName), pool);
705     }
706 
707     @Override
708     public HTableInterface getTable(TableName tableName, ExecutorService pool) throws IOException {
709       if (managed) {
710         throw new IOException("The connection has to be unmanaged.");
711       }
712       return new HTable(tableName, this, pool);
713     }
714 
715     private ExecutorService getBatchPool() {
716       if (batchPool == null) {
717         // shared HTable thread executor not yet initialized
718         synchronized (this) {
719           if (batchPool == null) {
720             int maxThreads = conf.getInt("hbase.hconnection.threads.max", 256);
721             int coreThreads = conf.getInt("hbase.hconnection.threads.core", 256);
722             if (maxThreads == 0) {
723               maxThreads = Runtime.getRuntime().availableProcessors() * 8;
724             }
725             if (coreThreads == 0) {
726               coreThreads = Runtime.getRuntime().availableProcessors() * 8;
727             }
728             long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60);
729             LinkedBlockingQueue<Runnable> workQueue =
730               new LinkedBlockingQueue<Runnable>(maxThreads *
731                 conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
732                   HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
733             ThreadPoolExecutor tpe = new ThreadPoolExecutor(
734                 coreThreads,
735                 maxThreads,
736                 keepAliveTime,
737                 TimeUnit.SECONDS,
738                 workQueue,
739                 Threads.newDaemonThreadFactory(toString() + "-shared-"));
740             tpe.allowCoreThreadTimeOut(true);
741             this.batchPool = tpe;
742           }
743           this.cleanupPool = true;
744         }
745       }
746       return this.batchPool;
747     }
748 
749     protected ExecutorService getCurrentBatchPool() {
750       return batchPool;
751     }
752 
753     private void shutdownBatchPool() {
754       if (this.cleanupPool && this.batchPool != null && !this.batchPool.isShutdown()) {
755         this.batchPool.shutdown();
756         try {
757           if (!this.batchPool.awaitTermination(10, TimeUnit.SECONDS)) {
758             this.batchPool.shutdownNow();
759           }
760         } catch (InterruptedException e) {
761           this.batchPool.shutdownNow();
762         }
763       }
764     }
765 
766     /**
767      * @return The cluster registry implementation to use.
768      * @throws IOException
769      */
770     private Registry setupRegistry() throws IOException {
771       String registryClass = this.conf.get("hbase.client.registry.impl",
772         ZooKeeperRegistry.class.getName());
773       Registry registry = null;
774       try {
775         registry = (Registry)Class.forName(registryClass).newInstance();
776       } catch (Throwable t) {
777         throw new IOException(t);
778       }
779       registry.init(this);
780       return registry;
781     }
782 
783     /**
784      * For tests only.
785      * @param rpcClient Client we should use instead.
786      * @return Previous rpcClient
787      */
788     RpcClient setRpcClient(final RpcClient rpcClient) {
789       RpcClient oldRpcClient = this.rpcClient;
790       this.rpcClient = rpcClient;
791       return oldRpcClient;
792     }
793 
794     /**
795      * An identifier that will remain the same for a given connection.
796      * @return
797      */
798     public String toString(){
799       return "hconnection-0x" + Integer.toHexString(hashCode());
800     }
801 
802     protected String clusterId = null;
803 
804     void retrieveClusterId() {
805       if (clusterId != null) return;
806       this.clusterId = this.registry.getClusterId();
807       if (clusterId == null) {
808         clusterId = HConstants.CLUSTER_ID_DEFAULT;
809         LOG.debug("clusterid came back null, using default " + clusterId);
810       }
811     }
812 
813     @Override
814     public Configuration getConfiguration() {
815       return this.conf;
816     }
817 
818     private void checkIfBaseNodeAvailable(ZooKeeperWatcher zkw)
819       throws MasterNotRunningException {
820       String errorMsg;
821       try {
822         if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {
823           errorMsg = "The node " + zkw.baseZNode+" is not in ZooKeeper. "
824             + "It should have been written by the master. "
825             + "Check the value configured in 'zookeeper.znode.parent'. "
826             + "There could be a mismatch with the one configured in the master.";
827           LOG.error(errorMsg);
828           throw new MasterNotRunningException(errorMsg);
829         }
830       } catch (KeeperException e) {
831         errorMsg = "Can't get connection to ZooKeeper: " + e.getMessage();
832         LOG.error(errorMsg);
833         throw new MasterNotRunningException(errorMsg, e);
834       }
835     }
836 
837     /**
838      * @return true if the master is running, throws an exception otherwise
839      * @throws MasterNotRunningException - if the master is not running
840      * @throws ZooKeeperConnectionException
841      */
842     @Override
843     public boolean isMasterRunning()
844     throws MasterNotRunningException, ZooKeeperConnectionException {
845       // When getting the master connection, we check it's running,
846       // so if there is no exception, it means we've been able to get a
847       // connection on a running master
848       MasterKeepAliveConnection m = getKeepAliveMasterService();
849       m.close();
850       return true;
851     }
852 
853     @Override
854     public HRegionLocation getRegionLocation(final TableName tableName,
855         final byte [] row, boolean reload)
856     throws IOException {
857       return reload? relocateRegion(tableName, row): locateRegion(tableName, row);
858     }
859 
860     @Override
861     public HRegionLocation getRegionLocation(final byte[] tableName,
862         final byte [] row, boolean reload)
863     throws IOException {
864       return getRegionLocation(TableName.valueOf(tableName), row, reload);
865     }
866 
867     @Override
868     public boolean isTableEnabled(TableName tableName) throws IOException {
869       return this.registry.isTableOnlineState(tableName, true);
870     }
871 
872     @Override
873     public boolean isTableEnabled(byte[] tableName) throws IOException {
874       return isTableEnabled(TableName.valueOf(tableName));
875     }
876 
877     @Override
878     public boolean isTableDisabled(TableName tableName) throws IOException {
879       return this.registry.isTableOnlineState(tableName, false);
880     }
881 
882     @Override
883     public boolean isTableDisabled(byte[] tableName) throws IOException {
884       return isTableDisabled(TableName.valueOf(tableName));
885     }
886 
887     @Override
888     public boolean isTableAvailable(final TableName tableName) throws IOException {
889       final AtomicBoolean available = new AtomicBoolean(true);
890       final AtomicInteger regionCount = new AtomicInteger(0);
891       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
892         @Override
893         public boolean processRow(Result row) throws IOException {
894           HRegionInfo info = MetaScanner.getHRegionInfo(row);
895           if (info != null && !info.isSplitParent()) {
896             if (tableName.equals(info.getTable())) {
897               ServerName server = HRegionInfo.getServerName(row);
898               if (server == null) {
899                 available.set(false);
900                 return false;
901               }
902               regionCount.incrementAndGet();
903             } else if (tableName.compareTo(info.getTable()) < 0) {
904               // Return if we are done with the current table
905               return false;
906             }
907           }
908           return true;
909         }
910       };
911       MetaScanner.metaScan(conf, this, visitor, tableName);
912       return available.get() && (regionCount.get() > 0);
913     }
914 
915     @Override
916     public boolean isTableAvailable(final byte[] tableName) throws IOException {
917       return isTableAvailable(TableName.valueOf(tableName));
918     }
919 
920     @Override
921     public boolean isTableAvailable(final TableName tableName, final byte[][] splitKeys)
922         throws IOException {
923       final AtomicBoolean available = new AtomicBoolean(true);
924       final AtomicInteger regionCount = new AtomicInteger(0);
925       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
926         @Override
927         public boolean processRow(Result row) throws IOException {
928           HRegionInfo info = MetaScanner.getHRegionInfo(row);
929           if (info != null && !info.isSplitParent()) {
930             if (tableName.equals(info.getTable())) {
931               ServerName server = HRegionInfo.getServerName(row);
932               if (server == null) {
933                 available.set(false);
934                 return false;
935               }
936               if (!Bytes.equals(info.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
937                 for (byte[] splitKey : splitKeys) {
938                   // Just check if the splitkey is available
939                   if (Bytes.equals(info.getStartKey(), splitKey)) {
940                     regionCount.incrementAndGet();
941                     break;
942                   }
943                 }
944               } else {
945                 // Always empty start row should be counted
946                 regionCount.incrementAndGet();
947               }
948             } else if (tableName.compareTo(info.getTable()) < 0) {
949               // Return if we are done with the current table
950               return false;
951             }
952           }
953           return true;
954         }
955       };
956       MetaScanner.metaScan(conf, this, visitor, tableName);
957       // +1 needs to be added so that the empty start row is also taken into account
958       return available.get() && (regionCount.get() == splitKeys.length + 1);
959     }
960 
961     @Override
962     public boolean isTableAvailable(final byte[] tableName, final byte[][] splitKeys)
963         throws IOException {
964       return isTableAvailable(TableName.valueOf(tableName), splitKeys);
965     }
966 
967     @Override
968     public HRegionLocation locateRegion(final byte[] regionName) throws IOException {
969       return locateRegion(HRegionInfo.getTable(regionName),
970           HRegionInfo.getStartKey(regionName), false, true);
971     }
972 
973     @Override
974     public boolean isDeadServer(ServerName sn) {
975       if (clusterStatusListener == null) {
976         return false;
977       } else {
978         return clusterStatusListener.isDeadServer(sn);
979       }
980     }
981 
982     @Override
983     public List<HRegionLocation> locateRegions(final TableName tableName)
984     throws IOException {
985       return locateRegions (tableName, false, true);
986     }
987 
988     @Override
989     public List<HRegionLocation> locateRegions(final byte[] tableName)
990     throws IOException {
991       return locateRegions(TableName.valueOf(tableName));
992     }
993 
994     @Override
995     public List<HRegionLocation> locateRegions(final TableName tableName,
996         final boolean useCache, final boolean offlined) throws IOException {
997       NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, this,
998           tableName, offlined);
999       final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();
1000       for (HRegionInfo regionInfo : regions.keySet()) {
1001         locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true));
1002       }
1003       return locations;
1004     }
1005 
1006     @Override
1007     public List<HRegionLocation> locateRegions(final byte[] tableName,
1008        final boolean useCache, final boolean offlined) throws IOException {
1009       return locateRegions(TableName.valueOf(tableName), useCache, offlined);
1010     }
1011 
1012     @Override
1013     public HRegionLocation locateRegion(final TableName tableName,
1014         final byte [] row)
1015     throws IOException{
1016       return locateRegion(tableName, row, true, true);
1017     }
1018 
1019     @Override
1020     public HRegionLocation locateRegion(final byte[] tableName,
1021         final byte [] row)
1022     throws IOException{
1023       return locateRegion(TableName.valueOf(tableName), row);
1024     }
1025 
1026     @Override
1027     public HRegionLocation relocateRegion(final TableName tableName,
1028         final byte [] row) throws IOException{
1029       // Since this is an explicit request not to use any caching, finding
1030       // disabled tables should not be desirable.  This will ensure that an exception is thrown when
1031       // the first time a disabled table is interacted with.
1032       if (isTableDisabled(tableName)) {
1033         throw new TableNotEnabledException(tableName.getNameAsString() + " is disabled.");
1034       }
1035 
1036       return locateRegion(tableName, row, false, true);
1037     }
1038 
1039     @Override
1040     public HRegionLocation relocateRegion(final byte[] tableName,
1041         final byte [] row) throws IOException {
1042       return relocateRegion(TableName.valueOf(tableName), row);
1043     }
1044 
1045 
1046     private HRegionLocation locateRegion(final TableName tableName,
1047       final byte [] row, boolean useCache, boolean retry)
1048     throws IOException {
1049       if (this.closed) throw new IOException(toString() + " closed");
1050       if (tableName== null || tableName.getName().length == 0) {
1051         throw new IllegalArgumentException(
1052             "table name cannot be null or zero length");
1053       }
1054 
1055       if (tableName.equals(TableName.META_TABLE_NAME)) {
1056         return this.registry.getMetaRegionLocation();
1057       } else {
1058         // Region not in the cache - have to go to the meta RS
1059         return locateRegionInMeta(TableName.META_TABLE_NAME, tableName, row,
1060           useCache, userRegionLock, retry);
1061       }
1062     }
1063 
1064     /*
1065      * Search hbase:meta for the HRegionLocation info that contains the table and
1066      * row we're seeking. It will prefetch certain number of regions info and
1067      * save them to the global region cache.
1068      */
1069     private void prefetchRegionCache(final TableName tableName,
1070         final byte[] row) {
1071       // Implement a new visitor for MetaScanner, and use it to walk through
1072       // the hbase:meta
1073       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
1074         public boolean processRow(Result result) throws IOException {
1075           try {
1076             HRegionInfo regionInfo = MetaScanner.getHRegionInfo(result);
1077             if (regionInfo == null) {
1078               return true;
1079             }
1080 
1081             // possible we got a region of a different table...
1082             if (!regionInfo.getTable().equals(tableName)) {
1083               return false; // stop scanning
1084             }
1085             if (regionInfo.isOffline()) {
1086               // don't cache offline regions
1087               return true;
1088             }
1089 
1090             ServerName serverName = HRegionInfo.getServerName(result);
1091             if (serverName == null) {
1092               return true; // don't cache it
1093             }
1094             // instantiate the location
1095             long seqNum = HRegionInfo.getSeqNumDuringOpen(result);
1096             HRegionLocation loc = new HRegionLocation(regionInfo, serverName, seqNum);
1097             // cache this meta entry
1098             cacheLocation(tableName, null, loc);
1099             return true;
1100           } catch (RuntimeException e) {
1101             throw new IOException(e);
1102           }
1103         }
1104       };
1105       try {
1106         // pre-fetch certain number of regions info at region cache.
1107         MetaScanner.metaScan(conf, this, visitor, tableName, row,
1108             this.prefetchRegionLimit, TableName.META_TABLE_NAME);
1109       } catch (IOException e) {
1110         LOG.warn("Encountered problems when prefetch hbase:meta table: ", e);
1111       }
1112     }
1113 
1114     /*
1115       * Search the hbase:meta table for the HRegionLocation
1116       * info that contains the table and row we're seeking.
1117       */
1118     private HRegionLocation locateRegionInMeta(final TableName parentTable,
1119       final TableName tableName, final byte [] row, boolean useCache,
1120       Object regionLockObject, boolean retry)
1121     throws IOException {
1122       HRegionLocation location;
1123       // If we are supposed to be using the cache, look in the cache to see if
1124       // we already have the region.
1125       if (useCache) {
1126         location = getCachedLocation(tableName, row);
1127         if (location != null) {
1128           return location;
1129         }
1130       }
1131       int localNumRetries = retry ? numTries : 1;
1132       // build the key of the meta region we should be looking for.
1133       // the extra 9's on the end are necessary to allow "exact" matches
1134       // without knowing the precise region names.
1135       byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
1136         HConstants.NINES, false);
1137       for (int tries = 0; true; tries++) {
1138         if (tries >= localNumRetries) {
1139           throw new NoServerForRegionException("Unable to find region for "
1140             + Bytes.toStringBinary(row) + " after " + numTries + " tries.");
1141         }
1142 
1143         HRegionLocation metaLocation = null;
1144         try {
1145           // locate the meta region
1146           metaLocation = locateRegion(parentTable, metaKey, true, false);
1147           // If null still, go around again.
1148           if (metaLocation == null) continue;
1149           ClientService.BlockingInterface service = getClient(metaLocation.getServerName());
1150 
1151           Result regionInfoRow;
1152           // This block guards against two threads trying to load the meta
1153           // region at the same time. The first will load the meta region and
1154           // the second will use the value that the first one found.
1155           if (useCache) {
1156             if (TableName.META_TABLE_NAME.equals(parentTable) &&
1157                 getRegionCachePrefetch(tableName)) {
1158               synchronized (regionLockObject) {
1159                 // Check the cache again for a hit in case some other thread made the
1160                 // same query while we were waiting on the lock.
1161                 location = getCachedLocation(tableName, row);
1162                 if (location != null) {
1163                   return location;
1164                 }
1165                 // If the parent table is META, we may want to pre-fetch some
1166                 // region info into the global region cache for this table.
1167                 prefetchRegionCache(tableName, row);
1168               }
1169             }
1170             location = getCachedLocation(tableName, row);
1171             if (location != null) {
1172               return location;
1173             }
1174           } else {
1175             // If we are not supposed to be using the cache, delete any existing cached location
1176             // so it won't interfere.
1177             forceDeleteCachedLocation(tableName, row);
1178           }
1179 
1180           // Query the meta region for the location of the meta region
1181           regionInfoRow = ProtobufUtil.getRowOrBefore(service,
1182               metaLocation.getRegionInfo().getRegionName(), metaKey,
1183               HConstants.CATALOG_FAMILY);
1184 
1185           if (regionInfoRow == null) {
1186             throw new TableNotFoundException(tableName);
1187           }
1188 
1189           // convert the row result into the HRegionLocation we need!
1190           HRegionInfo regionInfo = MetaScanner.getHRegionInfo(regionInfoRow);
1191           if (regionInfo == null) {
1192             throw new IOException("HRegionInfo was null or empty in " +
1193               parentTable + ", row=" + regionInfoRow);
1194           }
1195 
1196           // possible we got a region of a different table...
1197           if (!regionInfo.getTable().equals(tableName)) {
1198             throw new TableNotFoundException(
1199                   "Table '" + tableName + "' was not found, got: " +
1200                   regionInfo.getTable() + ".");
1201           }
1202           if (regionInfo.isSplit()) {
1203             throw new RegionOfflineException("the only available region for" +
1204               " the required row is a split parent," +
1205               " the daughters should be online soon: " +
1206               regionInfo.getRegionNameAsString());
1207           }
1208           if (regionInfo.isOffline()) {
1209             throw new RegionOfflineException("the region is offline, could" +
1210               " be caused by a disable table call: " +
1211               regionInfo.getRegionNameAsString());
1212           }
1213 
1214           ServerName serverName = HRegionInfo.getServerName(regionInfoRow);
1215           if (serverName == null) {
1216             throw new NoServerForRegionException("No serverName " +
1217               "in " + parentTable + " for " +
1218               regionInfo.getRegionNameAsString() + " containing " +
1219               Bytes.toStringBinary(row));
1220           }
1221 
1222           if (isDeadServer(serverName)){
1223             throw new RegionServerStoppedException("hbase:meta says the region "+
1224                 regionInfo.getRegionNameAsString()+" is managed by the server " + serverName +
1225                 ", but it is dead.");
1226           }
1227 
1228           // Instantiate the location
1229           location = new HRegionLocation(regionInfo, serverName,
1230             HRegionInfo.getSeqNumDuringOpen(regionInfoRow));
1231           cacheLocation(tableName, null, location);
1232           return location;
1233         } catch (TableNotFoundException e) {
1234           // if we got this error, probably means the table just plain doesn't
1235           // exist. rethrow the error immediately. this should always be coming
1236           // from the HTable constructor.
1237           throw e;
1238         } catch (IOException e) {
1239           if (e instanceof RemoteException) {
1240             e = ((RemoteException)e).unwrapRemoteException();
1241           }
1242           if (tries < numTries - 1) {
1243             if (LOG.isDebugEnabled()) {
1244               LOG.debug("locateRegionInMeta failed; parentTable=" + parentTable +
1245                 ", metaLocation=" + ((metaLocation == null)? "null": "{" + metaLocation + "}") +
1246                 ", attempt=" + tries + "/" + this.numTries + "; retrying after=" +
1247                 ConnectionUtils.getPauseTime(this.pause, tries) + "ms; because: " + e.getMessage());
1248             }
1249           } else {
1250             throw e;
1251           }
1252           // Only relocate the parent region if necessary
1253           if(!(e instanceof RegionOfflineException ||
1254               e instanceof NoServerForRegionException)) {
1255             relocateRegion(parentTable, metaKey);
1256           }
1257         }
1258         try{
1259           Thread.sleep(ConnectionUtils.getPauseTime(this.pause, tries));
1260         } catch (InterruptedException e) {
1261           Thread.currentThread().interrupt();
1262           throw new IOException("Giving up trying to location region in " +
1263             "meta: thread is interrupted.");
1264         }
1265       }
1266     }
1267 
1268     /*
1269      * Search the cache for a location that fits our table and row key.
1270      * Return null if no suitable region is located.
1271      *
1272      * @param tableName
1273      * @param row
1274      * @return Null or region location found in cache.
1275      */
1276     HRegionLocation getCachedLocation(final TableName tableName,
1277         final byte [] row) {
1278       ConcurrentSkipListMap<byte[], HRegionLocation> tableLocations =
1279         getTableLocations(tableName);
1280 
1281       Entry<byte[], HRegionLocation> e = tableLocations.floorEntry(row);
1282       if (e == null) {
1283         return null;
1284       }
1285       HRegionLocation possibleRegion = e.getValue();
1286 
1287       // make sure that the end key is greater than the row we're looking
1288       // for, otherwise the row actually belongs in the next region, not
1289       // this one. the exception case is when the endkey is
1290       // HConstants.EMPTY_END_ROW, signifying that the region we're
1291       // checking is actually the last region in the table.
1292       byte[] endKey = possibleRegion.getRegionInfo().getEndKey();
1293       if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
1294           tableName.getRowComparator().compareRows(
1295               endKey, 0, endKey.length, row, 0, row.length) > 0) {
1296         return possibleRegion;
1297       }
1298 
1299       // Passed all the way through, so we got nothing - complete cache miss
1300       return null;
1301     }
1302 
1303     /**
1304      * Delete a cached location, no matter what it is. Called when we were told to not use cache.
1305      * @param tableName tableName
1306      * @param row
1307      */
1308     void forceDeleteCachedLocation(final TableName tableName, final byte [] row) {
1309       HRegionLocation rl = null;
1310       Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1311       // start to examine the cache. we can only do cache actions
1312       // if there's something in the cache for this table.
1313       rl = getCachedLocation(tableName, row);
1314       if (rl != null) {
1315         tableLocations.remove(rl.getRegionInfo().getStartKey());
1316       }
1317       if ((rl != null) && LOG.isDebugEnabled()) {
1318         LOG.debug("Removed " + rl.getHostname() + ":" + rl.getPort()
1319           + " as a location of " + rl.getRegionInfo().getRegionNameAsString() +
1320           " for tableName=" + tableName + " from cache");
1321       }
1322     }
1323 
1324     /*
1325      * Delete all cached entries of a table that maps to a specific location.
1326      */
1327     @Override
1328     public void clearCaches(final ServerName serverName) {
1329       if (!this.cachedServers.contains(serverName)) {
1330         return;
1331       }
1332 
1333       boolean deletedSomething = false;
1334       synchronized (this.cachedServers) {
1335         // We block here, because if there is an error on a server, it's likely that multiple
1336         //  threads will get the error  simultaneously. If there are hundreds of thousand of
1337         //  region location to check, it's better to do this only once. A better pattern would
1338         //  be to check if the server is dead when we get the region location.
1339         if (!this.cachedServers.contains(serverName)) {
1340           return;
1341         }
1342         for (Map<byte[], HRegionLocation> tableLocations : cachedRegionLocations.values()) {
1343           for (Entry<byte[], HRegionLocation> e : tableLocations.entrySet()) {
1344             HRegionLocation value = e.getValue();
1345             if (value != null
1346                 && serverName.equals(value.getServerName())) {
1347               tableLocations.remove(e.getKey());
1348               deletedSomething = true;
1349             }
1350           }
1351         }
1352         this.cachedServers.remove(serverName);
1353       }
1354       if (deletedSomething && LOG.isDebugEnabled()) {
1355         LOG.debug("Removed all cached region locations that map to " + serverName);
1356       }
1357     }
1358 
1359     /*
1360      * @param tableName
1361      * @return Map of cached locations for passed <code>tableName</code>
1362      */
1363     private ConcurrentSkipListMap<byte[], HRegionLocation> getTableLocations(
1364         final TableName tableName) {
1365       // find the map of cached locations for this table
1366       ConcurrentSkipListMap<byte[], HRegionLocation> result;
1367       result = this.cachedRegionLocations.get(tableName);
1368       // if tableLocations for this table isn't built yet, make one
1369       if (result == null) {
1370         result = new ConcurrentSkipListMap<byte[], HRegionLocation>(Bytes.BYTES_COMPARATOR);
1371         ConcurrentSkipListMap<byte[], HRegionLocation> old =
1372             this.cachedRegionLocations.putIfAbsent(tableName, result);
1373         if (old != null) {
1374           return old;
1375         }
1376       }
1377       return result;
1378     }
1379 
1380     @Override
1381     public void clearRegionCache() {
1382       this.cachedRegionLocations.clear();
1383       this.cachedServers.clear();
1384     }
1385 
1386     @Override
1387     public void clearRegionCache(final TableName tableName) {
1388       this.cachedRegionLocations.remove(tableName);
1389     }
1390 
1391     @Override
1392     public void clearRegionCache(final byte[] tableName) {
1393       clearRegionCache(TableName.valueOf(tableName));
1394     }
1395 
1396     /**
1397      * Put a newly discovered HRegionLocation into the cache.
1398      * @param tableName The table name.
1399      * @param source the source of the new location, if it's not coming from meta
1400      * @param location the new location
1401      */
1402     private void cacheLocation(final TableName tableName, final HRegionLocation source,
1403         final HRegionLocation location) {
1404       boolean isFromMeta = (source == null);
1405       byte [] startKey = location.getRegionInfo().getStartKey();
1406       ConcurrentMap<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1407       HRegionLocation oldLocation = tableLocations.putIfAbsent(startKey, location);
1408       boolean isNewCacheEntry = (oldLocation == null);
1409       if (isNewCacheEntry) {
1410         cachedServers.add(location.getServerName());
1411         return;
1412       }
1413       boolean updateCache;
1414       // If the server in cache sends us a redirect, assume it's always valid.
1415       if (oldLocation.equals(source)) {
1416         updateCache = true;
1417       } else {
1418         long newLocationSeqNum = location.getSeqNum();
1419         // Meta record is stale - some (probably the same) server has closed the region
1420         // with later seqNum and told us about the new location.
1421         boolean isStaleMetaRecord = isFromMeta && (oldLocation.getSeqNum() > newLocationSeqNum);
1422         // Same as above for redirect. However, in this case, if the number is equal to previous
1423         // record, the most common case is that first the region was closed with seqNum, and then
1424         // opened with the same seqNum; hence we will ignore the redirect.
1425         // There are so many corner cases with various combinations of opens and closes that
1426         // an additional counter on top of seqNum would be necessary to handle them all.
1427         boolean isStaleRedirect = !isFromMeta && (oldLocation.getSeqNum() >= newLocationSeqNum);
1428         boolean isStaleUpdate = (isStaleMetaRecord || isStaleRedirect);
1429         updateCache = (!isStaleUpdate);
1430       }
1431       if (updateCache) {
1432         tableLocations.replace(startKey, oldLocation, location);
1433         cachedServers.add(location.getServerName());
1434       }
1435     }
1436 
1437     // Map keyed by service name + regionserver to service stub implementation
1438     private final ConcurrentHashMap<String, Object> stubs =
1439       new ConcurrentHashMap<String, Object>();
1440     // Map of locks used creating service stubs per regionserver.
1441     private final ConcurrentHashMap<String, String> connectionLock =
1442       new ConcurrentHashMap<String, String>();
1443 
1444     /**
1445      * State of the MasterService connection/setup.
1446      */
1447     static class MasterServiceState {
1448       HConnection connection;
1449       MasterService.BlockingInterface stub;
1450       int userCount;
1451       long keepAliveUntil = Long.MAX_VALUE;
1452 
1453       MasterServiceState (final HConnection connection) {
1454         super();
1455         this.connection = connection;
1456       }
1457 
1458       @Override
1459       public String toString() {
1460         return "MasterService";
1461       }
1462 
1463       Object getStub() {
1464         return this.stub;
1465       }
1466 
1467       void clearStub() {
1468         this.stub = null;
1469       }
1470 
1471       boolean isMasterRunning() throws ServiceException {
1472         IsMasterRunningResponse response =
1473           this.stub.isMasterRunning(null, RequestConverter.buildIsMasterRunningRequest());
1474         return response != null? response.getIsMasterRunning(): false;
1475       }
1476     }
1477 
1478     /**
1479      * Makes a client-side stub for master services. Sub-class to specialize.
1480      * Depends on hosting class so not static.  Exists so we avoid duplicating a bunch of code
1481      * when setting up the MasterMonitorService and MasterAdminService.
1482      */
1483     abstract class StubMaker {
1484       /**
1485        * Returns the name of the service stub being created.
1486        */
1487       protected abstract String getServiceName();
1488 
1489       /**
1490        * Make stub and cache it internal so can be used later doing the isMasterRunning call.
1491        * @param channel
1492        */
1493       protected abstract Object makeStub(final BlockingRpcChannel channel);
1494 
1495       /**
1496        * Once setup, check it works by doing isMasterRunning check.
1497        * @throws ServiceException
1498        */
1499       protected abstract void isMasterRunning() throws ServiceException;
1500 
1501       /**
1502        * Create a stub. Try once only.  It is not typed because there is no common type to
1503        * protobuf services nor their interfaces.  Let the caller do appropriate casting.
1504        * @return A stub for master services.
1505        * @throws IOException
1506        * @throws KeeperException
1507        * @throws ServiceException
1508        */
1509       private Object makeStubNoRetries() throws IOException, KeeperException, ServiceException {
1510         ZooKeeperKeepAliveConnection zkw;
1511         try {
1512           zkw = getKeepAliveZooKeeperWatcher();
1513         } catch (IOException e) {
1514           throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
1515         }
1516         try {
1517           checkIfBaseNodeAvailable(zkw);
1518           ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
1519           if (sn == null) {
1520             String msg = "ZooKeeper available but no active master location found";
1521             LOG.info(msg);
1522             throw new MasterNotRunningException(msg);
1523           }
1524           if (isDeadServer(sn)) {
1525             throw new MasterNotRunningException(sn + " is dead.");
1526           }
1527           // Use the security info interface name as our stub key
1528           String key = getStubKey(getServiceName(), sn.getHostAndPort());
1529           connectionLock.putIfAbsent(key, key);
1530           Object stub = null;
1531           synchronized (connectionLock.get(key)) {
1532             stub = stubs.get(key);
1533             if (stub == null) {
1534               BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sn,
1535                 user, rpcTimeout);
1536               stub = makeStub(channel);
1537               isMasterRunning();
1538               stubs.put(key, stub);
1539             }
1540           }
1541           return stub;
1542         } finally {
1543           zkw.close();
1544         }
1545       }
1546 
1547       /**
1548        * Create a stub against the master.  Retry if necessary.
1549        * @return A stub to do <code>intf</code> against the master
1550        * @throws MasterNotRunningException
1551        */
1552       @edu.umd.cs.findbugs.annotations.SuppressWarnings (value="SWL_SLEEP_WITH_LOCK_HELD")
1553       Object makeStub() throws MasterNotRunningException {
1554         // The lock must be at the beginning to prevent multiple master creations
1555         //  (and leaks) in a multithread context
1556         synchronized (masterAndZKLock) {
1557           Exception exceptionCaught = null;
1558           Object stub = null;
1559           int tries = 0;
1560           while (!closed && stub == null) {
1561             tries++;
1562             try {
1563               stub = makeStubNoRetries();
1564             } catch (IOException e) {
1565               exceptionCaught = e;
1566             } catch (KeeperException e) {
1567               exceptionCaught = e;
1568             } catch (ServiceException e) {
1569               exceptionCaught = e;
1570             }
1571 
1572             if (exceptionCaught != null)
1573               // It failed. If it's not the last try, we're going to wait a little
1574               if (tries < numTries) {
1575                 // tries at this point is 1 or more; decrement to start from 0.
1576                 long pauseTime = ConnectionUtils.getPauseTime(pause, tries - 1);
1577                 LOG.info("getMaster attempt " + tries + " of " + numTries +
1578                     " failed; retrying after sleep of " + pauseTime + ", exception=" +
1579                   exceptionCaught);
1580 
1581                 try {
1582                   Thread.sleep(pauseTime);
1583                 } catch (InterruptedException e) {
1584                   Thread.currentThread().interrupt();
1585                   throw new RuntimeException(
1586                       "Thread was interrupted while trying to connect to master.", e);
1587                 }
1588               } else {
1589                 // Enough tries, we stop now
1590                 LOG.info("getMaster attempt " + tries + " of " + numTries +
1591                     " failed; no more retrying.", exceptionCaught);
1592                 throw new MasterNotRunningException(exceptionCaught);
1593               }
1594           }
1595 
1596           if (stub == null) {
1597             // implies this.closed true
1598             throw new MasterNotRunningException("Connection was closed while trying to get master");
1599           }
1600           return stub;
1601         }
1602       }
1603     }
1604 
1605     /**
1606      * Class to make a MasterServiceStubMaker stub.
1607      */
1608     class MasterServiceStubMaker extends StubMaker {
1609       private MasterService.BlockingInterface stub;
1610       @Override
1611       protected String getServiceName() {
1612         return MasterService.getDescriptor().getName();
1613       }
1614 
1615       @Override
1616       @edu.umd.cs.findbugs.annotations.SuppressWarnings("SWL_SLEEP_WITH_LOCK_HELD")
1617       MasterService.BlockingInterface makeStub() throws MasterNotRunningException {
1618         return (MasterService.BlockingInterface)super.makeStub();
1619       }
1620 
1621       @Override
1622       protected Object makeStub(BlockingRpcChannel channel) {
1623         this.stub = MasterService.newBlockingStub(channel);
1624         return this.stub;
1625       }
1626 
1627       @Override
1628       protected void isMasterRunning() throws ServiceException {
1629         this.stub.isMasterRunning(null, RequestConverter.buildIsMasterRunningRequest());
1630       }
1631     }
1632 
1633     @Override
1634     public AdminService.BlockingInterface getAdmin(final ServerName serverName)
1635         throws IOException {
1636       return getAdmin(serverName, false);
1637     }
1638 
1639     @Override
1640     // Nothing is done w/ the 'master' parameter.  It is ignored.
1641     public AdminService.BlockingInterface getAdmin(final ServerName serverName,
1642       final boolean master)
1643     throws IOException {
1644       if (isDeadServer(serverName)) {
1645         throw new RegionServerStoppedException(serverName + " is dead.");
1646       }
1647       String key = getStubKey(AdminService.BlockingInterface.class.getName(),
1648         serverName.getHostAndPort());
1649       this.connectionLock.putIfAbsent(key, key);
1650       AdminService.BlockingInterface stub = null;
1651       synchronized (this.connectionLock.get(key)) {
1652         stub = (AdminService.BlockingInterface)this.stubs.get(key);
1653         if (stub == null) {
1654           BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(serverName,
1655             user, this.rpcTimeout);
1656           stub = AdminService.newBlockingStub(channel);
1657           this.stubs.put(key, stub);
1658         }
1659       }
1660       return stub;
1661     }
1662 
1663     @Override
1664     public ClientService.BlockingInterface getClient(final ServerName sn)
1665     throws IOException {
1666       if (isDeadServer(sn)) {
1667         throw new RegionServerStoppedException(sn + " is dead.");
1668       }
1669       String key = getStubKey(ClientService.BlockingInterface.class.getName(), sn.getHostAndPort());
1670       this.connectionLock.putIfAbsent(key, key);
1671       ClientService.BlockingInterface stub = null;
1672       synchronized (this.connectionLock.get(key)) {
1673         stub = (ClientService.BlockingInterface)this.stubs.get(key);
1674         if (stub == null) {
1675           BlockingRpcChannel channel = this.rpcClient.createBlockingRpcChannel(sn,
1676             user, this.rpcTimeout);
1677           stub = ClientService.newBlockingStub(channel);
1678           // In old days, after getting stub/proxy, we'd make a call.  We are not doing that here.
1679           // Just fail on first actual call rather than in here on setup.
1680           this.stubs.put(key, stub);
1681         }
1682       }
1683       return stub;
1684     }
1685 
1686     static String getStubKey(final String serviceName, final String rsHostnamePort) {
1687       return serviceName + "@" + rsHostnamePort;
1688     }
1689 
1690     private ZooKeeperKeepAliveConnection keepAliveZookeeper;
1691     private int keepAliveZookeeperUserCount;
1692     private boolean canCloseZKW = true;
1693 
1694     // keepAlive time, in ms. No reason to make it configurable.
1695     private static final long keepAlive = 5 * 60 * 1000;
1696 
1697     /**
1698      * Retrieve a shared ZooKeeperWatcher. You must close it it once you've have finished with it.
1699      * @return The shared instance. Never returns null.
1700      */
1701     ZooKeeperKeepAliveConnection getKeepAliveZooKeeperWatcher()
1702       throws IOException {
1703       synchronized (masterAndZKLock) {
1704         if (keepAliveZookeeper == null) {
1705           if (this.closed) {
1706             throw new IOException(toString() + " closed");
1707           }
1708           // We don't check that our link to ZooKeeper is still valid
1709           // But there is a retry mechanism in the ZooKeeperWatcher itself
1710           keepAliveZookeeper = new ZooKeeperKeepAliveConnection(conf, this.toString(), this);
1711         }
1712         keepAliveZookeeperUserCount++;
1713         keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1714         return keepAliveZookeeper;
1715       }
1716     }
1717 
1718     void releaseZooKeeperWatcher(final ZooKeeperWatcher zkw) {
1719       if (zkw == null){
1720         return;
1721       }
1722       synchronized (masterAndZKLock) {
1723         --keepAliveZookeeperUserCount;
1724         if (keepAliveZookeeperUserCount <= 0 ){
1725           keepZooKeeperWatcherAliveUntil = System.currentTimeMillis() + keepAlive;
1726         }
1727       }
1728     }
1729 
1730     /**
1731      * Creates a Chore thread to check the connections to master & zookeeper
1732      *  and close them when they reach their closing time (
1733      *  {@link MasterServiceState#keepAliveUntil} and
1734      *  {@link #keepZooKeeperWatcherAliveUntil}). Keep alive time is
1735      *  managed by the release functions and the variable {@link #keepAlive}
1736      */
1737     private static class DelayedClosing extends Chore implements Stoppable {
1738       private HConnectionImplementation hci;
1739       Stoppable stoppable;
1740 
1741       private DelayedClosing(
1742         HConnectionImplementation hci, Stoppable stoppable){
1743         super(
1744           "ZooKeeperWatcher and Master delayed closing for connection "+hci,
1745           60*1000, // We check every minutes
1746           stoppable);
1747         this.hci = hci;
1748         this.stoppable = stoppable;
1749       }
1750 
1751       static DelayedClosing createAndStart(HConnectionImplementation hci){
1752         Stoppable stoppable = new Stoppable() {
1753               private volatile boolean isStopped = false;
1754               @Override public void stop(String why) { isStopped = true;}
1755               @Override public boolean isStopped() {return isStopped;}
1756             };
1757 
1758         return new DelayedClosing(hci, stoppable);
1759       }
1760 
1761       protected void closeMasterProtocol(MasterServiceState protocolState) {
1762         if (System.currentTimeMillis() > protocolState.keepAliveUntil) {
1763           hci.closeMasterService(protocolState);
1764           protocolState.keepAliveUntil = Long.MAX_VALUE;
1765         }
1766       }
1767 
1768       @Override
1769       protected void chore() {
1770         synchronized (hci.masterAndZKLock) {
1771           if (hci.canCloseZKW) {
1772             if (System.currentTimeMillis() >
1773               hci.keepZooKeeperWatcherAliveUntil) {
1774 
1775               hci.closeZooKeeperWatcher();
1776               hci.keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1777             }
1778           }
1779           closeMasterProtocol(hci.masterServiceState);
1780           closeMasterProtocol(hci.masterServiceState);
1781         }
1782       }
1783 
1784       @Override
1785       public void stop(String why) {
1786         stoppable.stop(why);
1787       }
1788 
1789       @Override
1790       public boolean isStopped() {
1791         return stoppable.isStopped();
1792       }
1793     }
1794 
1795     private void closeZooKeeperWatcher() {
1796       synchronized (masterAndZKLock) {
1797         if (keepAliveZookeeper != null) {
1798           LOG.info("Closing zookeeper sessionid=0x" +
1799             Long.toHexString(
1800               keepAliveZookeeper.getRecoverableZooKeeper().getSessionId()));
1801           keepAliveZookeeper.internalClose();
1802           keepAliveZookeeper = null;
1803         }
1804         keepAliveZookeeperUserCount = 0;
1805       }
1806     }
1807 
1808     final MasterServiceState masterServiceState = new MasterServiceState(this);
1809 
1810     @Override
1811     public MasterService.BlockingInterface getMaster() throws MasterNotRunningException {
1812       return getKeepAliveMasterService();
1813     }
1814 
1815     private void resetMasterServiceState(final MasterServiceState mss) {
1816       mss.userCount++;
1817       mss.keepAliveUntil = Long.MAX_VALUE;
1818     }
1819 
1820     @Override
1821     public MasterKeepAliveConnection getKeepAliveMasterService()
1822     throws MasterNotRunningException {
1823       synchronized (masterAndZKLock) {
1824         if (!isKeepAliveMasterConnectedAndRunning(this.masterServiceState)) {
1825           MasterServiceStubMaker stubMaker = new MasterServiceStubMaker();
1826           this.masterServiceState.stub = stubMaker.makeStub();
1827         }
1828         resetMasterServiceState(this.masterServiceState);
1829       }
1830       // Ugly delegation just so we can add in a Close method.
1831       final MasterService.BlockingInterface stub = this.masterServiceState.stub;
1832       return new MasterKeepAliveConnection() {
1833         MasterServiceState mss = masterServiceState;
1834         @Override
1835         public AddColumnResponse addColumn(RpcController controller, AddColumnRequest request)
1836         throws ServiceException {
1837           return stub.addColumn(controller, request);
1838         }
1839 
1840         @Override
1841         public DeleteColumnResponse deleteColumn(RpcController controller,
1842             DeleteColumnRequest request)
1843         throws ServiceException {
1844           return stub.deleteColumn(controller, request);
1845         }
1846 
1847         @Override
1848         public ModifyColumnResponse modifyColumn(RpcController controller,
1849             ModifyColumnRequest request)
1850         throws ServiceException {
1851           return stub.modifyColumn(controller, request);
1852         }
1853 
1854         @Override
1855         public MoveRegionResponse moveRegion(RpcController controller,
1856             MoveRegionRequest request) throws ServiceException {
1857           return stub.moveRegion(controller, request);
1858         }
1859 
1860         @Override
1861         public DispatchMergingRegionsResponse dispatchMergingRegions(
1862             RpcController controller, DispatchMergingRegionsRequest request)
1863             throws ServiceException {
1864           return stub.dispatchMergingRegions(controller, request);
1865         }
1866 
1867         @Override
1868         public AssignRegionResponse assignRegion(RpcController controller,
1869             AssignRegionRequest request) throws ServiceException {
1870           return stub.assignRegion(controller, request);
1871         }
1872 
1873         @Override
1874         public UnassignRegionResponse unassignRegion(RpcController controller,
1875             UnassignRegionRequest request) throws ServiceException {
1876           return stub.unassignRegion(controller, request);
1877         }
1878 
1879         @Override
1880         public OfflineRegionResponse offlineRegion(RpcController controller,
1881             OfflineRegionRequest request) throws ServiceException {
1882           return stub.offlineRegion(controller, request);
1883         }
1884 
1885         @Override
1886         public DeleteTableResponse deleteTable(RpcController controller,
1887             DeleteTableRequest request) throws ServiceException {
1888           return stub.deleteTable(controller, request);
1889         }
1890 
1891         @Override
1892         public EnableTableResponse enableTable(RpcController controller,
1893             EnableTableRequest request) throws ServiceException {
1894           return stub.enableTable(controller, request);
1895         }
1896 
1897         @Override
1898         public DisableTableResponse disableTable(RpcController controller,
1899             DisableTableRequest request) throws ServiceException {
1900           return stub.disableTable(controller, request);
1901         }
1902 
1903         @Override
1904         public ModifyTableResponse modifyTable(RpcController controller,
1905             ModifyTableRequest request) throws ServiceException {
1906           return stub.modifyTable(controller, request);
1907         }
1908 
1909         @Override
1910         public CreateTableResponse createTable(RpcController controller,
1911             CreateTableRequest request) throws ServiceException {
1912           return stub.createTable(controller, request);
1913         }
1914 
1915         @Override
1916         public ShutdownResponse shutdown(RpcController controller,
1917             ShutdownRequest request) throws ServiceException {
1918           return stub.shutdown(controller, request);
1919         }
1920 
1921         @Override
1922         public StopMasterResponse stopMaster(RpcController controller,
1923             StopMasterRequest request) throws ServiceException {
1924           return stub.stopMaster(controller, request);
1925         }
1926 
1927         @Override
1928         public BalanceResponse balance(RpcController controller,
1929             BalanceRequest request) throws ServiceException {
1930           return stub.balance(controller, request);
1931         }
1932 
1933         @Override
1934         public SetBalancerRunningResponse setBalancerRunning(
1935             RpcController controller, SetBalancerRunningRequest request)
1936             throws ServiceException {
1937           return stub.setBalancerRunning(controller, request);
1938         }
1939 
1940         @Override
1941         public RunCatalogScanResponse runCatalogScan(RpcController controller,
1942             RunCatalogScanRequest request) throws ServiceException {
1943           return stub.runCatalogScan(controller, request);
1944         }
1945 
1946         @Override
1947         public EnableCatalogJanitorResponse enableCatalogJanitor(
1948             RpcController controller, EnableCatalogJanitorRequest request)
1949             throws ServiceException {
1950           return stub.enableCatalogJanitor(controller, request);
1951         }
1952 
1953         @Override
1954         public IsCatalogJanitorEnabledResponse isCatalogJanitorEnabled(
1955             RpcController controller, IsCatalogJanitorEnabledRequest request)
1956             throws ServiceException {
1957           return stub.isCatalogJanitorEnabled(controller, request);
1958         }
1959 
1960         @Override
1961         public CoprocessorServiceResponse execMasterService(
1962             RpcController controller, CoprocessorServiceRequest request)
1963             throws ServiceException {
1964           return stub.execMasterService(controller, request);
1965         }
1966 
1967         @Override
1968         public SnapshotResponse snapshot(RpcController controller,
1969             SnapshotRequest request) throws ServiceException {
1970           return stub.snapshot(controller, request);
1971         }
1972 
1973         @Override
1974         public GetCompletedSnapshotsResponse getCompletedSnapshots(
1975             RpcController controller, GetCompletedSnapshotsRequest request)
1976             throws ServiceException {
1977           return stub.getCompletedSnapshots(controller, request);
1978         }
1979 
1980         @Override
1981         public DeleteSnapshotResponse deleteSnapshot(RpcController controller,
1982             DeleteSnapshotRequest request) throws ServiceException {
1983           return stub.deleteSnapshot(controller, request);
1984         }
1985 
1986         @Override
1987         public IsSnapshotDoneResponse isSnapshotDone(RpcController controller,
1988             IsSnapshotDoneRequest request) throws ServiceException {
1989           return stub.isSnapshotDone(controller, request);
1990         }
1991 
1992         @Override
1993         public RestoreSnapshotResponse restoreSnapshot(
1994             RpcController controller, RestoreSnapshotRequest request)
1995             throws ServiceException {
1996           return stub.restoreSnapshot(controller, request);
1997         }
1998 
1999         @Override
2000         public IsRestoreSnapshotDoneResponse isRestoreSnapshotDone(
2001             RpcController controller, IsRestoreSnapshotDoneRequest request)
2002             throws ServiceException {
2003           return stub.isRestoreSnapshotDone(controller, request);
2004         }
2005 
2006         @Override
2007         public IsMasterRunningResponse isMasterRunning(
2008             RpcController controller, IsMasterRunningRequest request)
2009             throws ServiceException {
2010           return stub.isMasterRunning(controller, request);
2011         }
2012 
2013         @Override
2014         public ModifyNamespaceResponse modifyNamespace(RpcController controller,
2015             ModifyNamespaceRequest request)
2016         throws ServiceException {
2017           return stub.modifyNamespace(controller, request);
2018         }
2019 
2020         @Override
2021         public CreateNamespaceResponse createNamespace(RpcController controller, CreateNamespaceRequest request) throws ServiceException {
2022           return stub.createNamespace(controller, request);
2023         }
2024 
2025         @Override
2026         public DeleteNamespaceResponse deleteNamespace(RpcController controller, DeleteNamespaceRequest request) throws ServiceException {
2027           return stub.deleteNamespace(controller, request);
2028         }
2029 
2030         @Override
2031         public GetNamespaceDescriptorResponse getNamespaceDescriptor(RpcController controller, GetNamespaceDescriptorRequest request) throws ServiceException {
2032           return stub.getNamespaceDescriptor(controller, request);
2033         }
2034 
2035         @Override
2036         public ListNamespaceDescriptorsResponse listNamespaceDescriptors(RpcController controller, ListNamespaceDescriptorsRequest request) throws ServiceException {
2037           return stub.listNamespaceDescriptors(controller, request);
2038         }
2039 
2040         @Override
2041         public ListTableDescriptorsByNamespaceResponse listTableDescriptorsByNamespace(RpcController controller, ListTableDescriptorsByNamespaceRequest request) throws ServiceException {
2042           return stub.listTableDescriptorsByNamespace(controller, request);
2043         }
2044 
2045         @Override
2046         public ListTableNamesByNamespaceResponse listTableNamesByNamespace(RpcController controller,
2047               ListTableNamesByNamespaceRequest request) throws ServiceException {
2048           return stub.listTableNamesByNamespace(controller, request);
2049         }
2050 
2051         @Override
2052         public void close() {
2053           release(this.mss);
2054         }
2055 
2056         @Override
2057         public GetSchemaAlterStatusResponse getSchemaAlterStatus(
2058             RpcController controller, GetSchemaAlterStatusRequest request)
2059             throws ServiceException {
2060           return stub.getSchemaAlterStatus(controller, request);
2061         }
2062 
2063         @Override
2064         public GetTableDescriptorsResponse getTableDescriptors(
2065             RpcController controller, GetTableDescriptorsRequest request)
2066             throws ServiceException {
2067           return stub.getTableDescriptors(controller, request);
2068         }
2069 
2070         @Override
2071         public GetTableNamesResponse getTableNames(
2072             RpcController controller, GetTableNamesRequest request)
2073             throws ServiceException {
2074           return stub.getTableNames(controller, request);
2075         }
2076 
2077         @Override
2078         public GetClusterStatusResponse getClusterStatus(
2079             RpcController controller, GetClusterStatusRequest request)
2080             throws ServiceException {
2081           return stub.getClusterStatus(controller, request);
2082         }
2083       };
2084     }
2085  
2086 
2087     private static void release(MasterServiceState mss) {
2088       if (mss != null && mss.connection != null) {
2089         ((HConnectionImplementation)mss.connection).releaseMaster(mss);
2090       }
2091     }
2092 
2093     private boolean isKeepAliveMasterConnectedAndRunning(MasterServiceState mss) {
2094       if (mss.getStub() == null){
2095         return false;
2096       }
2097       try {
2098         return mss.isMasterRunning();
2099       } catch (UndeclaredThrowableException e) {
2100         // It's somehow messy, but we can receive exceptions such as
2101         //  java.net.ConnectException but they're not declared. So we catch it...
2102         LOG.info("Master connection is not running anymore", e.getUndeclaredThrowable());
2103         return false;
2104       } catch (ServiceException se) {
2105         LOG.warn("Checking master connection", se);
2106         return false;
2107       }
2108     }
2109 
2110     void releaseMaster(MasterServiceState mss) {
2111       if (mss.getStub() == null) return;
2112       synchronized (masterAndZKLock) {
2113         --mss.userCount;
2114         if (mss.userCount <= 0) {
2115           mss.keepAliveUntil = System.currentTimeMillis() + keepAlive;
2116         }
2117       }
2118     }
2119 
2120     private void closeMasterService(MasterServiceState mss) {
2121       if (mss.getStub() != null) {
2122         LOG.info("Closing master protocol: " + mss);
2123         mss.clearStub();
2124       }
2125       mss.userCount = 0;
2126     }
2127 
2128     /**
2129      * Immediate close of the shared master. Can be by the delayed close or when closing the
2130      * connection itself.
2131      */
2132     private void closeMaster() {
2133       synchronized (masterAndZKLock) {
2134         closeMasterService(masterServiceState);
2135       }
2136     }
2137 
2138     void updateCachedLocation(HRegionInfo hri, HRegionLocation source,
2139                               ServerName serverName, long seqNum) {
2140       HRegionLocation newHrl = new HRegionLocation(hri, serverName, seqNum);
2141       cacheLocation(hri.getTable(), source, newHrl);
2142     }
2143 
2144    /**
2145     * Deletes the cached location of the region if necessary, based on some error from source.
2146     * @param hri The region in question.
2147     * @param source The source of the error that prompts us to invalidate cache.
2148     */
2149    void deleteCachedLocation(HRegionInfo hri, HRegionLocation source) {
2150      ConcurrentMap<byte[], HRegionLocation> tableLocations = getTableLocations(hri.getTable());
2151      tableLocations.remove(hri.getStartKey(), source);
2152    }
2153 
2154     @Override
2155     public void deleteCachedRegionLocation(final HRegionLocation location) {
2156       if (location == null) {
2157         return;
2158       }
2159 
2160       HRegionLocation removedLocation;
2161       TableName tableName = location.getRegionInfo().getTable();
2162       Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
2163       removedLocation = tableLocations.remove(location.getRegionInfo().getStartKey());
2164       if (LOG.isDebugEnabled() && removedLocation != null) {
2165         LOG.debug("Removed " +
2166             location.getRegionInfo().getRegionNameAsString() +
2167             " for tableName=" + tableName +
2168             " from cache");
2169       }
2170     }
2171 
2172     /**
2173      * Update the location with the new value (if the exception is a RegionMovedException)
2174      * or delete it from the cache. Does nothing if we can be sure from the exception that
2175      * the location is still accurate, or if the cache has already been updated.
2176      * @param exception an object (to simplify user code) on which we will try to find a nested
2177      *                  or wrapped or both RegionMovedException
2178      * @param source server that is the source of the location update.
2179      */
2180     @Override
2181     public void updateCachedLocations(final TableName tableName, byte[] rowkey,
2182       final Object exception, final HRegionLocation source) {
2183       if (rowkey == null || tableName == null) {
2184         LOG.warn("Coding error, see method javadoc. row=" + (rowkey == null ? "null" : rowkey) +
2185             ", tableName=" + (tableName == null ? "null" : tableName));
2186         return;
2187       }
2188 
2189       if (source == null || source.getServerName() == null){
2190         // This should not happen, but let's secure ourselves.
2191         return;
2192       }
2193 
2194       // Is it something we have already updated?
2195       final HRegionLocation oldLocation = getCachedLocation(tableName, rowkey);
2196       if (oldLocation == null || !source.getServerName().equals(oldLocation.getServerName())) {
2197         // There is no such location in the cache (it's been removed already) or
2198         // the cache has already been refreshed with a different location.  => nothing to do
2199         return;
2200       }
2201 
2202       HRegionInfo regionInfo = oldLocation.getRegionInfo();
2203       Throwable cause = findException(exception);
2204       if (cause != null) {
2205         if (cause instanceof RegionTooBusyException || cause instanceof RegionOpeningException) {
2206           // We know that the region is still on this region server
2207           return;
2208         }
2209 
2210         if (cause instanceof RegionMovedException) {
2211           RegionMovedException rme = (RegionMovedException) cause;
2212           if (LOG.isTraceEnabled()) {
2213             LOG.trace("Region " + regionInfo.getRegionNameAsString() + " moved to " +
2214                 rme.getHostname() + ":" + rme.getPort() +
2215                 " according to " + source.getHostnamePort());
2216           }
2217           // We know that the region is not anymore on this region server, but we know
2218           //  the new location.
2219           updateCachedLocation(
2220               regionInfo, source, rme.getServerName(), rme.getLocationSeqNum());
2221           return;
2222         }
2223       }
2224 
2225       // If we're here, it means that can cannot be sure about the location, so we remove it from
2226       //  the cache.
2227       deleteCachedLocation(regionInfo, source);
2228     }
2229 
2230     @Override
2231     public void updateCachedLocations(final byte[] tableName, byte[] rowkey,
2232       final Object exception, final HRegionLocation source) {
2233       updateCachedLocations(TableName.valueOf(tableName), rowkey, exception, source);
2234     }
2235 
2236     @Override
2237     @Deprecated
2238     public void processBatch(List<? extends Row> list,
2239         final TableName tableName,
2240         ExecutorService pool,
2241         Object[] results) throws IOException, InterruptedException {
2242       // This belongs in HTable!!! Not in here.  St.Ack
2243 
2244       // results must be the same size as list
2245       if (results.length != list.size()) {
2246         throw new IllegalArgumentException(
2247           "argument results must be the same size as argument list");
2248       }
2249       processBatchCallback(list, tableName, pool, results, null);
2250     }
2251 
2252     @Override
2253     @Deprecated
2254     public void processBatch(List<? extends Row> list,
2255         final byte[] tableName,
2256         ExecutorService pool,
2257         Object[] results) throws IOException, InterruptedException {
2258       processBatch(list, TableName.valueOf(tableName), pool, results);
2259     }
2260 
2261     /**
2262      * Send the queries in parallel on the different region servers. Retries on failures.
2263      * If the method returns it means that there is no error, and the 'results' array will
2264      * contain no exception. On error, an exception is thrown, and the 'results' array will
2265      * contain results and exceptions.
2266      * @deprecated since 0.96 - Use {@link HTable#processBatchCallback} instead
2267      */
2268     @Override
2269     @Deprecated
2270     public <R> void processBatchCallback(
2271       List<? extends Row> list,
2272       TableName tableName,
2273       ExecutorService pool,
2274       Object[] results,
2275       Batch.Callback<R> callback)
2276       throws IOException, InterruptedException {
2277 
2278       // To fulfill the original contract, we have a special callback. This callback
2279       //  will set the results in the Object array.
2280       ObjectResultFiller<R> cb = new ObjectResultFiller<R>(results, callback);
2281       AsyncProcess<?> asyncProcess = createAsyncProcess(tableName, pool, cb, conf);
2282 
2283       // We're doing a submit all. This way, the originalIndex will match the initial list.
2284       asyncProcess.submitAll(list);
2285       asyncProcess.waitUntilDone();
2286 
2287       if (asyncProcess.hasError()) {
2288         throw asyncProcess.getErrors();
2289       }
2290     }
2291 
2292     @Override
2293     @Deprecated
2294     public <R> void processBatchCallback(
2295       List<? extends Row> list,
2296       byte[] tableName,
2297       ExecutorService pool,
2298       Object[] results,
2299       Batch.Callback<R> callback)
2300       throws IOException, InterruptedException {
2301       processBatchCallback(list, TableName.valueOf(tableName), pool, results, callback);
2302     }
2303 
2304     // For tests.
2305     protected <R> AsyncProcess createAsyncProcess(TableName tableName, ExecutorService pool,
2306            AsyncProcess.AsyncProcessCallback<R> callback, Configuration conf) {
2307       return new AsyncProcess<R>(this, tableName, pool, callback, conf,
2308           RpcRetryingCallerFactory.instantiate(conf));
2309     }
2310 
2311 
2312     /**
2313      * Fill the result array for the interfaces using it.
2314      */
2315     private static class ObjectResultFiller<Res>
2316         implements AsyncProcess.AsyncProcessCallback<Res> {
2317 
2318       private final Object[] results;
2319       private Batch.Callback<Res> callback;
2320 
2321       ObjectResultFiller(Object[] results, Batch.Callback<Res> callback) {
2322         this.results = results;
2323         this.callback = callback;
2324       }
2325 
2326       @Override
2327       public void success(int pos, byte[] region, Row row, Res result) {
2328         assert pos < results.length;
2329         results[pos] = result;
2330         if (callback != null) {
2331           callback.update(region, row.getRow(), result);
2332         }
2333       }
2334 
2335       @Override
2336       public boolean failure(int pos, byte[] region, Row row, Throwable t) {
2337         assert pos < results.length;
2338         results[pos] = t;
2339         //Batch.Callback<Res> was not called on failure in 0.94. We keep this.
2340         return true; // we want to have this failure in the failures list.
2341       }
2342 
2343       @Override
2344       public boolean retriableFailure(int originalIndex, Row row, byte[] region,
2345                                       Throwable exception) {
2346         return true; // we retry
2347       }
2348     }
2349 
2350 
2351     /*
2352      * Return the number of cached region for a table. It will only be called
2353      * from a unit test.
2354      */
2355     int getNumberOfCachedRegionLocations(final TableName tableName) {
2356       Map<byte[], HRegionLocation> tableLocs = this.cachedRegionLocations.get(tableName);
2357       if (tableLocs == null) {
2358         return 0;
2359       }
2360       return tableLocs.values().size();
2361     }
2362 
2363     /**
2364      * Check the region cache to see whether a region is cached yet or not.
2365      * Called by unit tests.
2366      * @param tableName tableName
2367      * @param row row
2368      * @return Region cached or not.
2369      */
2370     boolean isRegionCached(TableName tableName, final byte[] row) {
2371       HRegionLocation location = getCachedLocation(tableName, row);
2372       return location != null;
2373     }
2374 
2375     @Override
2376     public void setRegionCachePrefetch(final TableName tableName,
2377         final boolean enable) {
2378       if (!enable) {
2379         regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName.getName()));
2380       }
2381       else {
2382         regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName.getName()));
2383       }
2384     }
2385 
2386     @Override
2387     public void setRegionCachePrefetch(final byte[] tableName,
2388         final boolean enable) {
2389       setRegionCachePrefetch(TableName.valueOf(tableName), enable);
2390     }
2391 
2392     @Override
2393     public boolean getRegionCachePrefetch(TableName tableName) {
2394       return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName.getName()));
2395     }
2396 
2397     @Override
2398     public boolean getRegionCachePrefetch(byte[] tableName) {
2399       return getRegionCachePrefetch(TableName.valueOf(tableName));
2400     }
2401 
2402     @Override
2403     public void abort(final String msg, Throwable t) {
2404       if (t instanceof KeeperException.SessionExpiredException
2405         && keepAliveZookeeper != null) {
2406         synchronized (masterAndZKLock) {
2407           if (keepAliveZookeeper != null) {
2408             LOG.warn("This client just lost it's session with ZooKeeper," +
2409               " closing it." +
2410               " It will be recreated next time someone needs it", t);
2411             closeZooKeeperWatcher();
2412           }
2413         }
2414       } else {
2415         if (t != null) {
2416           LOG.fatal(msg, t);
2417         } else {
2418           LOG.fatal(msg);
2419         }
2420         this.aborted = true;
2421         close();
2422         this.closed = true;
2423       }
2424     }
2425 
2426     @Override
2427     public boolean isClosed() {
2428       return this.closed;
2429     }
2430 
2431     @Override
2432     public boolean isAborted(){
2433       return this.aborted;
2434     }
2435 
2436     @Override
2437     public int getCurrentNrHRS() throws IOException {
2438       return this.registry.getCurrentNrHRS();
2439     }
2440 
2441     /**
2442      * Increment this client's reference count.
2443      */
2444     void incCount() {
2445       ++refCount;
2446     }
2447 
2448     /**
2449      * Decrement this client's reference count.
2450      */
2451     void decCount() {
2452       if (refCount > 0) {
2453         --refCount;
2454       }
2455     }
2456 
2457     /**
2458      * Return if this client has no reference
2459      *
2460      * @return true if this client has no reference; false otherwise
2461      */
2462     boolean isZeroReference() {
2463       return refCount == 0;
2464     }
2465 
2466     void internalClose() {
2467       if (this.closed) {
2468         return;
2469       }
2470       delayedClosing.stop("Closing connection");
2471       closeMaster();
2472       shutdownBatchPool();
2473       this.closed = true;
2474       closeZooKeeperWatcher();
2475       this.stubs.clear();
2476       if (clusterStatusListener != null) {
2477         clusterStatusListener.close();
2478       }
2479       if (rpcClient != null) {
2480         rpcClient.stop();
2481       }
2482     }
2483 
2484     @Override
2485     public void close() {
2486       if (managed) {
2487         if (aborted) {
2488           HConnectionManager.deleteStaleConnection(this);
2489         } else {
2490           HConnectionManager.deleteConnection(this, false);
2491         }
2492       } else {
2493         internalClose();
2494       }
2495     }
2496 
2497     /**
2498      * Close the connection for good, regardless of what the current value of
2499      * {@link #refCount} is. Ideally, {@link #refCount} should be zero at this
2500      * point, which would be the case if all of its consumers close the
2501      * connection. However, on the off chance that someone is unable to close
2502      * the connection, perhaps because it bailed out prematurely, the method
2503      * below will ensure that this {@link HConnection} instance is cleaned up.
2504      * Caveat: The JVM may take an unknown amount of time to call finalize on an
2505      * unreachable object, so our hope is that every consumer cleans up after
2506      * itself, like any good citizen.
2507      */
2508     @Override
2509     protected void finalize() throws Throwable {
2510       super.finalize();
2511       // Pretend as if we are about to release the last remaining reference
2512       refCount = 1;
2513       close();
2514     }
2515 
2516     @Override
2517     public HTableDescriptor[] listTables() throws IOException {
2518       MasterKeepAliveConnection master = getKeepAliveMasterService();
2519       try {
2520         GetTableDescriptorsRequest req =
2521           RequestConverter.buildGetTableDescriptorsRequest((List<TableName>)null);
2522         return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2523       } catch (ServiceException se) {
2524         throw ProtobufUtil.getRemoteException(se);
2525       } finally {
2526         master.close();
2527       }
2528     }
2529 
2530     @Override
2531     public String[] getTableNames() throws IOException {
2532       TableName[] tableNames = listTableNames();
2533       String result[] = new String[tableNames.length];
2534       for (int i = 0; i < tableNames.length; i++) {
2535         result[i] = tableNames[i].getNameAsString();
2536       }
2537       return result;
2538     }
2539 
2540     @Override
2541     public TableName[] listTableNames() throws IOException {
2542       MasterKeepAliveConnection master = getKeepAliveMasterService();
2543       try {
2544         return ProtobufUtil.getTableNameArray(master.getTableNames(null,
2545             GetTableNamesRequest.newBuilder().build())
2546           .getTableNamesList());
2547       } catch (ServiceException se) {
2548         throw ProtobufUtil.getRemoteException(se);
2549       } finally {
2550         master.close();
2551       }
2552     }
2553 
2554     @Override
2555     public HTableDescriptor[] getHTableDescriptorsByTableName(
2556         List<TableName> tableNames) throws IOException {
2557       if (tableNames == null || tableNames.isEmpty()) return new HTableDescriptor[0];
2558       MasterKeepAliveConnection master = getKeepAliveMasterService();
2559       try {
2560         GetTableDescriptorsRequest req =
2561           RequestConverter.buildGetTableDescriptorsRequest(tableNames);
2562         return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2563       } catch (ServiceException se) {
2564         throw ProtobufUtil.getRemoteException(se);
2565       } finally {
2566         master.close();
2567       }
2568     }
2569 
2570     @Override
2571     public HTableDescriptor[] getHTableDescriptors(
2572         List<String> names) throws IOException {
2573       List<TableName> tableNames = new ArrayList(names.size());
2574       for(String name : names) {
2575         tableNames.add(TableName.valueOf(name));
2576       }
2577 
2578       return getHTableDescriptorsByTableName(tableNames);
2579     }
2580 
2581     /**
2582      * Connects to the master to get the table descriptor.
2583      * @param tableName table name
2584      * @return
2585      * @throws IOException if the connection to master fails or if the table
2586      *  is not found.
2587      */
2588     @Override
2589     public HTableDescriptor getHTableDescriptor(final TableName tableName)
2590     throws IOException {
2591       if (tableName == null) return null;
2592       if (tableName.equals(TableName.META_TABLE_NAME)) {
2593         return HTableDescriptor.META_TABLEDESC;
2594       }
2595       MasterKeepAliveConnection master = getKeepAliveMasterService();
2596       GetTableDescriptorsResponse htds;
2597       try {
2598         GetTableDescriptorsRequest req =
2599           RequestConverter.buildGetTableDescriptorsRequest(tableName);
2600         htds = master.getTableDescriptors(null, req);
2601       } catch (ServiceException se) {
2602         throw ProtobufUtil.getRemoteException(se);
2603       } finally {
2604         master.close();
2605       }
2606       if (!htds.getTableSchemaList().isEmpty()) {
2607         return HTableDescriptor.convert(htds.getTableSchemaList().get(0));
2608       }
2609       throw new TableNotFoundException(tableName.getNameAsString());
2610     }
2611 
2612     @Override
2613     public HTableDescriptor getHTableDescriptor(final byte[] tableName)
2614     throws IOException {
2615       return getHTableDescriptor(TableName.valueOf(tableName));
2616     }
2617   }
2618 
2619   /**
2620    * The record of errors for servers.
2621    */
2622   static class ServerErrorTracker {
2623     // We need a concurrent map here, as we could have multiple threads updating it in parallel.
2624     private final ConcurrentMap<HRegionLocation, ServerErrors> errorsByServer =
2625         new ConcurrentHashMap<HRegionLocation, ServerErrors>();
2626     private final long canRetryUntil;
2627     private final int maxRetries;
2628     private final String startTrackingTime;
2629 
2630     public ServerErrorTracker(long timeout, int maxRetries) {
2631       this.maxRetries = maxRetries;
2632       this.canRetryUntil = EnvironmentEdgeManager.currentTimeMillis() + timeout;
2633       this.startTrackingTime = new Date().toString();
2634     }
2635 
2636     /**
2637      * We stop to retry when we have exhausted BOTH the number of retries and the time allocated.
2638      */
2639     boolean canRetryMore(int numRetry) {
2640       // If there is a single try we must not take into account the time.
2641       return numRetry < maxRetries || (maxRetries > 1 &&
2642           EnvironmentEdgeManager.currentTimeMillis() < this.canRetryUntil);
2643     }
2644 
2645     /**
2646      * Calculates the back-off time for a retrying request to a particular server.
2647      *
2648      * @param server    The server in question.
2649      * @param basePause The default hci pause.
2650      * @return The time to wait before sending next request.
2651      */
2652     long calculateBackoffTime(HRegionLocation server, long basePause) {
2653       long result;
2654       ServerErrors errorStats = errorsByServer.get(server);
2655       if (errorStats != null) {
2656         result = ConnectionUtils.getPauseTime(basePause, errorStats.retries.get());
2657       } else {
2658         result = 0; // yes, if the server is not in our list we don't wait before retrying.
2659       }
2660       return result;
2661     }
2662 
2663     /**
2664      * Reports that there was an error on the server to do whatever bean-counting necessary.
2665      *
2666      * @param server The server in question.
2667      */
2668     void reportServerError(HRegionLocation server) {
2669       ServerErrors errors = errorsByServer.get(server);
2670       if (errors != null) {
2671         errors.addError();
2672       } else {
2673         errors = errorsByServer.putIfAbsent(server, new ServerErrors());
2674         if (errors != null){
2675           errors.addError();
2676         }
2677       }
2678     }
2679 
2680     String getStartTrackingTime() {
2681       return startTrackingTime;
2682     }
2683 
2684     /**
2685      * The record of errors for a server.
2686      */
2687     private static class ServerErrors {
2688       public final AtomicInteger retries = new AtomicInteger(0);
2689 
2690       public void addError() {
2691         retries.incrementAndGet();
2692       }
2693     }
2694   }
2695 
2696   /**
2697    * Look for an exception we know in the remote exception:
2698    * - hadoop.ipc wrapped exceptions
2699    * - nested exceptions
2700    * 
2701    * Looks for: RegionMovedException / RegionOpeningException / RegionTooBusyException
2702    * @return null if we didn't find the exception, the exception otherwise.
2703    */
2704   public static Throwable findException(Object exception) {
2705     if (exception == null || !(exception instanceof Throwable)) {
2706       return null;
2707     }
2708     Throwable cur = (Throwable) exception;
2709     while (cur != null) {
2710       if (cur instanceof RegionMovedException || cur instanceof RegionOpeningException
2711           || cur instanceof RegionTooBusyException) {
2712         return cur;
2713       }
2714       if (cur instanceof RemoteException) {
2715         RemoteException re = (RemoteException) cur;
2716         cur = re.unwrapRemoteException(
2717             RegionOpeningException.class, RegionMovedException.class,
2718             RegionTooBusyException.class);
2719         if (cur == null) {
2720           cur = re.unwrapRemoteException();
2721         }
2722         // unwrapRemoteException can return the exception given as a parameter when it cannot
2723         //  unwrap it. In this case, there is no need to look further
2724         // noinspection ObjectEquality
2725         if (cur == re) {
2726           return null;
2727         }
2728       } else {
2729         cur = cur.getCause();
2730       }
2731     }
2732 
2733     return null;
2734   }
2735 
2736   /**
2737    * Set the number of retries to use serverside when trying to communicate
2738    * with another server over {@link HConnection}.  Used updating catalog
2739    * tables, etc.  Call this method before we create any Connections.
2740    * @param c The Configuration instance to set the retries into.
2741    * @param log Used to log what we set in here.
2742    */
2743   public static void setServerSideHConnectionRetries(final Configuration c, final String sn,
2744       final Log log) {
2745     int hcRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2746       HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2747     // Go big.  Multiply by 10.  If we can't get to meta after this many retries
2748     // then something seriously wrong.
2749     int serversideMultiplier = c.getInt("hbase.client.serverside.retries.multiplier", 10);
2750     int retries = hcRetries * serversideMultiplier;
2751     c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, retries);
2752     log.debug(sn + " HConnection server-to-server retries=" + retries);
2753   }
2754 }