View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.lang.reflect.InvocationHandler;
24  import java.lang.reflect.InvocationTargetException;
25  import java.lang.reflect.Method;
26  import java.lang.reflect.Proxy;
27  import java.lang.reflect.UndeclaredThrowableException;
28  import java.net.InetSocketAddress;
29  import java.util.ArrayList;
30  import java.util.Collections;
31  import java.util.HashMap;
32  import java.util.HashSet;
33  import java.util.Iterator;
34  import java.util.LinkedHashMap;
35  import java.util.LinkedList;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.NavigableMap;
40  import java.util.Set;
41  import java.util.concurrent.Callable;
42  import java.util.concurrent.ConcurrentHashMap;
43  import java.util.concurrent.CopyOnWriteArraySet;
44  import java.util.concurrent.ExecutionException;
45  import java.util.concurrent.ExecutorService;
46  import java.util.concurrent.Future;
47  import java.util.concurrent.atomic.AtomicBoolean;
48  import java.util.concurrent.atomic.AtomicInteger;
49  
50  import org.apache.commons.logging.Log;
51  import org.apache.commons.logging.LogFactory;
52  import org.apache.hadoop.classification.InterfaceAudience;
53  import org.apache.hadoop.classification.InterfaceStability;
54  import org.apache.hadoop.conf.Configuration;
55  import org.apache.hadoop.hbase.Chore;
56  import org.apache.hadoop.hbase.HBaseConfiguration;
57  import org.apache.hadoop.hbase.HConstants;
58  import org.apache.hadoop.hbase.HRegionInfo;
59  import org.apache.hadoop.hbase.HRegionLocation;
60  import org.apache.hadoop.hbase.HTableDescriptor;
61  import org.apache.hadoop.hbase.IpcProtocol;
62  import org.apache.hadoop.hbase.KeyValue;
63  import org.apache.hadoop.hbase.MasterAdminProtocol;
64  import org.apache.hadoop.hbase.MasterMonitorProtocol;
65  import org.apache.hadoop.hbase.MasterProtocol;
66  import org.apache.hadoop.hbase.RemoteExceptionHandler;
67  import org.apache.hadoop.hbase.ServerName;
68  import org.apache.hadoop.hbase.Stoppable;
69  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
70  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
71  import org.apache.hadoop.hbase.client.coprocessor.Batch;
72  import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
73  import org.apache.hadoop.hbase.exceptions.MasterNotRunningException;
74  import org.apache.hadoop.hbase.exceptions.RegionMovedException;
75  import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
76  import org.apache.hadoop.hbase.exceptions.RegionServerStoppedException;
77  import org.apache.hadoop.hbase.exceptions.TableNotFoundException;
78  import org.apache.hadoop.hbase.exceptions.ZooKeeperConnectionException;
79  import org.apache.hadoop.hbase.ipc.HBaseClientRPC;
80  import org.apache.hadoop.hbase.ipc.ProtobufRpcClientEngine;
81  import org.apache.hadoop.hbase.ipc.RpcClientEngine;
82  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
83  import org.apache.hadoop.hbase.protobuf.RequestConverter;
84  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
85  import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetTableDescriptorsRequest;
86  import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetTableDescriptorsResponse;
87  import org.apache.hadoop.hbase.security.User;
88  import org.apache.hadoop.hbase.util.Addressing;
89  import org.apache.hadoop.hbase.util.Bytes;
90  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
91  import org.apache.hadoop.hbase.util.Pair;
92  import org.apache.hadoop.hbase.util.SoftValueSortedMap;
93  import org.apache.hadoop.hbase.util.Triple;
94  import org.apache.hadoop.hbase.zookeeper.*;
95  import org.apache.hadoop.ipc.RemoteException;
96  import org.apache.zookeeper.KeeperException;
97  
98  import com.google.protobuf.ServiceException;
99  
100 /**
101  * A non-instantiable class that manages {@link HConnection}s.
102  * This class has a static Map of {@link HConnection} instances keyed by
103  * {@link Configuration}; all invocations of {@link #getConnection(Configuration)}
104  * that pass the same {@link Configuration} instance will be returned the same
105  * {@link  HConnection} instance (Adding properties to a Configuration
106  * instance does not change its object identity).  Sharing {@link HConnection}
107  * instances is usually what you want; all clients of the {@link HConnection}
108  * instances share the HConnections' cache of Region locations rather than each
109  * having to discover for itself the location of meta, etc.  It makes
110  * sense for the likes of the pool of HTables class {@link HTablePool}, for
111  * instance (If concerned that a single {@link HConnection} is insufficient
112  * for sharing amongst clients in say an heavily-multithreaded environment,
113  * in practise its not proven to be an issue.  Besides, {@link HConnection} is
114  * implemented atop Hadoop RPC and as of this writing, Hadoop RPC does a
115  * connection per cluster-member, exclusively).
116  *
117  * <p>But sharing connections
118  * makes clean up of {@link HConnection} instances a little awkward.  Currently,
119  * clients cleanup by calling
120  * {@link #deleteConnection(Configuration)}.  This will shutdown the
121  * zookeeper connection the HConnection was using and clean up all
122  * HConnection resources as well as stopping proxies to servers out on the
123  * cluster. Not running the cleanup will not end the world; it'll
124  * just stall the closeup some and spew some zookeeper connection failed
125  * messages into the log.  Running the cleanup on a {@link HConnection} that is
126  * subsequently used by another will cause breakage so be careful running
127  * cleanup.
128  * <p>To create a {@link HConnection} that is not shared by others, you can
129  * create a new {@link Configuration} instance, pass this new instance to
130  * {@link #getConnection(Configuration)}, and then when done, close it up by
131  * doing something like the following:
132  * <pre>
133  * {@code
134  * Configuration newConfig = new Configuration(originalConf);
135  * HConnection connection = HConnectionManager.getConnection(newConfig);
136  * // Use the connection to your hearts' delight and then when done...
137  * HConnectionManager.deleteConnection(newConfig, true);
138  * }
139  * </pre>
140  * <p>Cleanup used to be done inside in a shutdown hook.  On startup we'd
141  * register a shutdown hook that called {@link #deleteAllConnections()}
142  * on its way out but the order in which shutdown hooks run is not defined so
143  * were problematic for clients of HConnection that wanted to register their
144  * own shutdown hooks so we removed ours though this shifts the onus for
145  * cleanup to the client.
146  */
147 @SuppressWarnings("serial")
148 @InterfaceAudience.Public
149 @InterfaceStability.Evolving
150 public class HConnectionManager {
151   // An LRU Map of HConnectionKey -> HConnection (TableServer).  All
152   // access must be synchronized.  This map is not private because tests
153   // need to be able to tinker with it.
154   static final Map<HConnectionKey, HConnectionImplementation> HBASE_INSTANCES;
155 
156   public static final int MAX_CACHED_HBASE_INSTANCES;
157 
158   /** Parameter name for what client protocol to use. */
159   public static final String CLIENT_PROTOCOL_CLASS = "hbase.clientprotocol.class";
160 
161   /** Default client protocol class name. */
162   public static final String DEFAULT_CLIENT_PROTOCOL_CLASS = ClientProtocol.class.getName();
163 
164   /** Parameter name for what admin protocol to use. */
165   public static final String REGION_PROTOCOL_CLASS = "hbase.adminprotocol.class";
166 
167   /** Default admin protocol class name. */
168   public static final String DEFAULT_ADMIN_PROTOCOL_CLASS = AdminProtocol.class.getName();
169 
170   private static final Log LOG = LogFactory.getLog(HConnectionManager.class);
171 
172   static {
173     // We set instances to one more than the value specified for {@link
174     // HConstants#ZOOKEEPER_MAX_CLIENT_CNXNS}. By default, the zk default max
175     // connections to the ensemble from the one client is 30, so in that case we
176     // should run into zk issues before the LRU hit this value of 31.
177     MAX_CACHED_HBASE_INSTANCES = HBaseConfiguration.create().getInt(
178         HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
179         HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS) + 1;
180     HBASE_INSTANCES = new LinkedHashMap<HConnectionKey, HConnectionImplementation>(
181         (int) (MAX_CACHED_HBASE_INSTANCES / 0.75F) + 1, 0.75F, true) {
182        @Override
183       protected boolean removeEldestEntry(
184           Map.Entry<HConnectionKey, HConnectionImplementation> eldest) {
185          return size() > MAX_CACHED_HBASE_INSTANCES;
186        }
187     };
188   }
189 
190   /*
191    * Non-instantiable.
192    */
193   protected HConnectionManager() {
194     super();
195   }
196 
197   /**
198    * Get the connection that goes with the passed <code>conf</code>
199    * configuration instance.
200    * If no current connection exists, method creates a new connection for the
201    * passed <code>conf</code> instance.
202    * @param conf configuration
203    * @return HConnection object for <code>conf</code>
204    * @throws ZooKeeperConnectionException
205    */
206   public static HConnection getConnection(Configuration conf)
207   throws IOException {
208     HConnectionKey connectionKey = new HConnectionKey(conf);
209     synchronized (HBASE_INSTANCES) {
210       HConnectionImplementation connection = HBASE_INSTANCES.get(connectionKey);
211       if (connection == null) {
212         connection = new HConnectionImplementation(conf, true);
213         HBASE_INSTANCES.put(connectionKey, connection);
214       } else if (connection.isClosed()) {
215         HConnectionManager.deleteConnection(connectionKey, true);
216         connection = new HConnectionImplementation(conf, true);
217         HBASE_INSTANCES.put(connectionKey, connection);
218       }
219       connection.incCount();
220       return connection;
221     }
222   }
223 
224   /**
225    * Create a new HConnection instance using the passed <code>conf</code>
226    * instance.
227    * Note: This bypasses the usual HConnection life cycle management!
228    * Use this with caution, the caller is responsible for closing the
229    * created connection.
230    * @param conf configuration
231    * @return HConnection object for <code>conf</code>
232    * @throws ZooKeeperConnectionException
233    */
234   public static HConnection createConnection(Configuration conf)
235   throws IOException {
236     return new HConnectionImplementation(conf, false);
237   }
238 
239   /**
240    * Delete connection information for the instance specified by configuration.
241    * If there are no more references to it, this will then close connection to
242    * the zookeeper ensemble and let go of all resources.
243    *
244    * @param conf
245    *          configuration whose identity is used to find {@link HConnection}
246    *          instance.
247    */
248   public static void deleteConnection(Configuration conf) {
249     deleteConnection(new HConnectionKey(conf), false);
250   }
251 
252   /**
253    * Delete stale connection information for the instance specified by configuration.
254    * This will then close connection to
255    * the zookeeper ensemble and let go of all resources.
256    *
257    * @param connection
258    */
259   public static void deleteStaleConnection(HConnection connection) {
260     deleteConnection(connection, true);
261   }
262 
263   /**
264    * Delete information for all connections.
265    */
266   public static void deleteAllConnections() {
267     synchronized (HBASE_INSTANCES) {
268       Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
269       connectionKeys.addAll(HBASE_INSTANCES.keySet());
270       for (HConnectionKey connectionKey : connectionKeys) {
271         deleteConnection(connectionKey, false);
272       }
273       HBASE_INSTANCES.clear();
274     }
275   }
276 
277   private static void deleteConnection(HConnection connection, boolean staleConnection) {
278     synchronized (HBASE_INSTANCES) {
279       for (Entry<HConnectionKey, HConnectionImplementation> connectionEntry : HBASE_INSTANCES
280           .entrySet()) {
281         if (connectionEntry.getValue() == connection) {
282           deleteConnection(connectionEntry.getKey(), staleConnection);
283           break;
284         }
285       }
286     }
287   }
288 
289   private static void deleteConnection(HConnectionKey connectionKey, boolean staleConnection) {
290     synchronized (HBASE_INSTANCES) {
291       HConnectionImplementation connection = HBASE_INSTANCES
292           .get(connectionKey);
293       if (connection != null) {
294         connection.decCount();
295         if (connection.isZeroReference() || staleConnection) {
296           HBASE_INSTANCES.remove(connectionKey);
297           connection.internalClose();
298         }
299       } else {
300         LOG.error("Connection not found in the list, can't delete it "+
301           "(connection key="+connectionKey+"). May be the key was modified?");
302       }
303     }
304   }
305 
306   /**
307    * It is provided for unit test cases which verify the behavior of region
308    * location cache prefetch.
309    * @return Number of cached regions for the table.
310    * @throws ZooKeeperConnectionException
311    */
312   static int getCachedRegionCount(Configuration conf,
313       final byte[] tableName)
314   throws IOException {
315     return execute(new HConnectable<Integer>(conf) {
316       @Override
317       public Integer connect(HConnection connection) {
318         return ((HConnectionImplementation) connection)
319             .getNumberOfCachedRegionLocations(tableName);
320       }
321     });
322   }
323 
324   /**
325    * It's provided for unit test cases which verify the behavior of region
326    * location cache prefetch.
327    * @return true if the region where the table and row reside is cached.
328    * @throws ZooKeeperConnectionException
329    */
330   static boolean isRegionCached(Configuration conf,
331       final byte[] tableName, final byte[] row) throws IOException {
332     return execute(new HConnectable<Boolean>(conf) {
333       @Override
334       public Boolean connect(HConnection connection) {
335         return ((HConnectionImplementation) connection).isRegionCached(tableName, row);
336       }
337     });
338   }
339 
340   /**
341    * This class makes it convenient for one to execute a command in the context
342    * of a {@link HConnection} instance based on the given {@link Configuration}.
343    *
344    * <p>
345    * If you find yourself wanting to use a {@link HConnection} for a relatively
346    * short duration of time, and do not want to deal with the hassle of creating
347    * and cleaning up that resource, then you should consider using this
348    * convenience class.
349    *
350    * @param <T>
351    *          the return type of the {@link HConnectable#connect(HConnection)}
352    *          method.
353    */
354   public static abstract class HConnectable<T> {
355     public Configuration conf;
356 
357     protected HConnectable(Configuration conf) {
358       this.conf = conf;
359     }
360 
361     public abstract T connect(HConnection connection) throws IOException;
362   }
363 
364   /**
365    * This convenience method invokes the given {@link HConnectable#connect}
366    * implementation using a {@link HConnection} instance that lasts just for the
367    * duration of that invocation.
368    *
369    * @param <T> the return type of the connect method
370    * @param connectable the {@link HConnectable} instance
371    * @return the value returned by the connect method
372    * @throws IOException
373    */
374   public static <T> T execute(HConnectable<T> connectable) throws IOException {
375     if (connectable == null || connectable.conf == null) {
376       return null;
377     }
378     Configuration conf = connectable.conf;
379     HConnection connection = HConnectionManager.getConnection(conf);
380     boolean connectSucceeded = false;
381     try {
382       T returnValue = connectable.connect(connection);
383       connectSucceeded = true;
384       return returnValue;
385     } finally {
386       try {
387         connection.close();
388       } catch (Exception e) {
389         if (connectSucceeded) {
390           throw new IOException("The connection to " + connection
391               + " could not be deleted.", e);
392         }
393       }
394     }
395   }
396 
397   /**
398    * Denotes a unique key to a {@link HConnection} instance.
399    *
400    * In essence, this class captures the properties in {@link Configuration}
401    * that may be used in the process of establishing a connection. In light of
402    * that, if any new such properties are introduced into the mix, they must be
403    * added to the {@link HConnectionKey#properties} list.
404    *
405    */
406   public static class HConnectionKey {
407     final static String[] CONNECTION_PROPERTIES = new String[] {
408         HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT,
409         HConstants.ZOOKEEPER_CLIENT_PORT,
410         HConstants.ZOOKEEPER_RECOVERABLE_WAITTIME,
411         HConstants.HBASE_CLIENT_PAUSE, HConstants.HBASE_CLIENT_RETRIES_NUMBER,
412         HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS,
413         HConstants.HBASE_RPC_TIMEOUT_KEY,
414         HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
415         HConstants.HBASE_META_SCANNER_CACHING,
416         HConstants.HBASE_CLIENT_INSTANCE_ID };
417 
418     private Map<String, String> properties;
419     private String username;
420 
421     public HConnectionKey(Configuration conf) {
422       Map<String, String> m = new HashMap<String, String>();
423       if (conf != null) {
424         for (String property : CONNECTION_PROPERTIES) {
425           String value = conf.get(property);
426           if (value != null) {
427             m.put(property, value);
428           }
429         }
430       }
431       this.properties = Collections.unmodifiableMap(m);
432 
433       try {
434         User currentUser = User.getCurrent();
435         if (currentUser != null) {
436           username = currentUser.getName();
437         }
438       } catch (IOException ioe) {
439         LOG.warn("Error obtaining current user, skipping username in HConnectionKey",
440             ioe);
441       }
442     }
443 
444     @Override
445     public int hashCode() {
446       final int prime = 31;
447       int result = 1;
448       if (username != null) {
449         result = username.hashCode();
450       }
451       for (String property : CONNECTION_PROPERTIES) {
452         String value = properties.get(property);
453         if (value != null) {
454           result = prime * result + value.hashCode();
455         }
456       }
457 
458       return result;
459     }
460 
461 
462     @edu.umd.cs.findbugs.annotations.SuppressWarnings (value="ES_COMPARING_STRINGS_WITH_EQ",
463         justification="Optimization")
464     @Override
465     public boolean equals(Object obj) {
466       if (this == obj)
467         return true;
468       if (obj == null)
469         return false;
470       if (getClass() != obj.getClass())
471         return false;
472       HConnectionKey that = (HConnectionKey) obj;
473       if (this.username != null && !this.username.equals(that.username)) {
474         return false;
475       } else if (this.username == null && that.username != null) {
476         return false;
477       }
478       if (this.properties == null) {
479         if (that.properties != null) {
480           return false;
481         }
482       } else {
483         if (that.properties == null) {
484           return false;
485         }
486         for (String property : CONNECTION_PROPERTIES) {
487           String thisValue = this.properties.get(property);
488           String thatValue = that.properties.get(property);
489           //noinspection StringEquality
490           if (thisValue == thatValue) {
491             continue;
492           }
493           if (thisValue == null || !thisValue.equals(thatValue)) {
494             return false;
495           }
496         }
497       }
498       return true;
499     }
500 
501     @Override
502     public String toString() {
503       return "HConnectionKey{" +
504         "properties=" + properties +
505         ", username='" + username + '\'' +
506         '}';
507     }
508   }
509 
510   /** Encapsulates connection to zookeeper and regionservers.*/
511   static class HConnectionImplementation implements HConnection, Closeable {
512     static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);
513     private final Class<? extends AdminProtocol> adminClass;
514     private final Class<? extends ClientProtocol> clientClass;
515     private final long pause;
516     private final int numRetries;
517     private final int maxRPCAttempts;
518     private final int rpcTimeout;
519     private final int prefetchRegionLimit;
520 
521     private volatile boolean closed;
522     private volatile boolean aborted;
523 
524     // package protected for the tests
525     ClusterStatusListener clusterStatusListener;
526 
527     private final Object userRegionLock = new Object();
528 
529     // We have a single lock for master & zk to prevent deadlocks. Having
530     //  one lock for ZK and one lock for master is not possible:
531     //  When creating a connection to master, we need a connection to ZK to get
532     //  its address. But another thread could have taken the ZK lock, and could
533     //  be waiting for the master lock => deadlock.
534     private final Object masterAndZKLock = new Object();
535 
536     private long keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
537     private final DelayedClosing delayedClosing =
538       DelayedClosing.createAndStart(this);
539 
540 
541     private final Configuration conf;
542 
543     // client RPC
544     private RpcClientEngine rpcEngine;
545 
546     // Known region ServerName.toString() -> RegionClient/Admin
547     private final ConcurrentHashMap<String, Map<String, IpcProtocol>> servers =
548       new ConcurrentHashMap<String, Map<String, IpcProtocol>>();
549     private final ConcurrentHashMap<String, String> connectionLock =
550       new ConcurrentHashMap<String, String>();
551 
552     /**
553      * Map of table to table {@link HRegionLocation}s.  The table key is made
554      * by doing a {@link Bytes#mapKey(byte[])} of the table's name.
555      */
556     private final Map<Integer, SoftValueSortedMap<byte [], HRegionLocation>>
557       cachedRegionLocations =
558         new HashMap<Integer, SoftValueSortedMap<byte [], HRegionLocation>>();
559 
560     // The presence of a server in the map implies it's likely that there is an
561     // entry in cachedRegionLocations that map to this server; but the absence
562     // of a server in this map guarentees that there is no entry in cache that
563     // maps to the absent server.
564     // The access to this attribute must be protected by a lock on cachedRegionLocations
565     private final Set<ServerName> cachedServers = new HashSet<ServerName>();
566 
567     // region cache prefetch is enabled by default. this set contains all
568     // tables whose region cache prefetch are disabled.
569     private final Set<Integer> regionCachePrefetchDisabledTables =
570       new CopyOnWriteArraySet<Integer>();
571 
572     private int refCount;
573 
574     // indicates whether this connection's life cycle is managed (by us)
575     private final boolean managed;
576     /**
577      * constructor
578      * @param conf Configuration object
579      */
580     @SuppressWarnings("unchecked")
581     public HConnectionImplementation(Configuration conf, boolean managed) throws IOException {
582       this.conf = conf;
583       this.managed = managed;
584       String adminClassName = conf.get(REGION_PROTOCOL_CLASS,
585         DEFAULT_ADMIN_PROTOCOL_CLASS);
586       this.closed = false;
587       try {
588         this.adminClass =
589           (Class<? extends AdminProtocol>) Class.forName(adminClassName);
590       } catch (ClassNotFoundException e) {
591         throw new UnsupportedOperationException(
592             "Unable to find region server interface " + adminClassName, e);
593       }
594       String clientClassName = conf.get(CLIENT_PROTOCOL_CLASS,
595         DEFAULT_CLIENT_PROTOCOL_CLASS);
596       try {
597         this.clientClass =
598           (Class<? extends ClientProtocol>) Class.forName(clientClassName);
599       } catch (ClassNotFoundException e) {
600         throw new UnsupportedOperationException(
601             "Unable to find client protocol " + clientClassName, e);
602       }
603       this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
604           HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
605       this.numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
606           HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
607       this.maxRPCAttempts = conf.getInt(
608           HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS,
609           HConstants.DEFAULT_HBASE_CLIENT_RPC_MAXATTEMPTS);
610       this.rpcTimeout = conf.getInt(
611           HConstants.HBASE_RPC_TIMEOUT_KEY,
612           HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
613       this.prefetchRegionLimit = conf.getInt(
614           HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
615           HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
616 
617       retrieveClusterId();
618 
619       // ProtobufRpcClientEngine is the main RpcClientEngine implementation,
620       // but we maintain access through an interface to allow overriding for tests
621       // RPC engine setup must follow obtaining the cluster ID for token authentication to work
622       this.rpcEngine = new ProtobufRpcClientEngine(this.conf, this.clusterId);
623 
624 
625       // Do we publish the status?
626       Class<? extends ClusterStatusListener.Listener> listenerClass =
627           conf.getClass(ClusterStatusListener.STATUS_LISTENER_CLASS,
628               ClusterStatusListener.DEFAULT_STATUS_LISTENER_CLASS,
629               ClusterStatusListener.Listener.class);
630 
631       if (listenerClass != null) {
632         clusterStatusListener = new ClusterStatusListener(
633             new ClusterStatusListener.DeadServerHandler() {
634               @Override
635               public void newDead(ServerName sn) {
636                 clearCaches(sn);
637                 rpcEngine.getClient().cancelConnections(sn.getHostname(), sn.getPort(), null);
638               }
639             }, conf, listenerClass);
640       }
641     }
642 
643     /**
644      * An identifier that will remain the same for a given connection.
645      * @return
646      */
647     public String toString(){
648       return "hconnection-0x" + Integer.toHexString(hashCode());
649     }
650 
651     private String clusterId = null;
652 
653     public final void retrieveClusterId(){
654       if (clusterId != null) {
655         return;
656       }
657 
658       // No synchronized here, worse case we will retrieve it twice, that's
659       //  not an issue.
660       ZooKeeperKeepAliveConnection zkw = null;
661       try {
662         zkw = getKeepAliveZooKeeperWatcher();
663         clusterId = ZKClusterId.readClusterIdZNode(zkw);
664         if (clusterId == null) {
665           LOG.info("ClusterId read in ZooKeeper is null");
666         }
667       } catch (KeeperException e) {
668         LOG.warn("Can't retrieve clusterId from Zookeeper", e);
669       } catch (IOException e) {
670         LOG.warn("Can't retrieve clusterId from Zookeeper", e);
671       } finally {
672         if (zkw != null) {
673           zkw.close();
674         }
675       }
676       if (clusterId == null) {
677         clusterId = HConstants.CLUSTER_ID_DEFAULT;
678       }
679 
680       LOG.info("ClusterId is " + clusterId);
681     }
682 
683     @Override
684     public Configuration getConfiguration() {
685       return this.conf;
686     }
687 
688     private static class MasterProtocolState {
689       public MasterProtocol protocol;
690       public int userCount;
691       public long keepAliveUntil = Long.MAX_VALUE;
692       public final Class<? extends MasterProtocol> protocolClass;
693 
694       public MasterProtocolState (
695           final Class<? extends MasterProtocol> protocolClass) {
696         this.protocolClass = protocolClass;
697       }
698     }
699 
700     /**
701      * Create a new Master proxy. Try once only.
702      */
703     private MasterProtocol createMasterInterface(
704         MasterProtocolState masterProtocolState)
705         throws IOException, KeeperException, ServiceException {
706 
707       ZooKeeperKeepAliveConnection zkw;
708       try {
709         zkw = getKeepAliveZooKeeperWatcher();
710       } catch (IOException e) {
711         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
712       }
713 
714       try {
715 
716         checkIfBaseNodeAvailable(zkw);
717         ServerName sn = MasterAddressTracker.getMasterAddress(zkw);
718         if (sn == null) {
719           String msg =
720             "ZooKeeper available but no active master location found";
721           LOG.info(msg);
722           throw new MasterNotRunningException(msg);
723         }
724 
725 
726         InetSocketAddress isa =
727           new InetSocketAddress(sn.getHostname(), sn.getPort());
728         MasterProtocol tryMaster = rpcEngine.getProxy(
729             masterProtocolState.protocolClass,
730             isa, this.conf, this.rpcTimeout);
731 
732         if (tryMaster.isMasterRunning(
733             null, RequestConverter.buildIsMasterRunningRequest()).getIsMasterRunning()) {
734           return tryMaster;
735         } else {
736           String msg = "Can create a proxy to master, but it is not running";
737           LOG.info(msg);
738           throw new MasterNotRunningException(msg);
739         }
740       } finally {
741         zkw.close();
742       }
743     }
744 
745     /**
746      * Create a master, retries if necessary.
747      */
748     @edu.umd.cs.findbugs.annotations.SuppressWarnings (value="SWL_SLEEP_WITH_LOCK_HELD")
749     private MasterProtocol createMasterWithRetries(
750       MasterProtocolState masterProtocolState) throws MasterNotRunningException {
751 
752       // The lock must be at the beginning to prevent multiple master creation
753       //  (and leaks) in a multithread context
754 
755       synchronized (this.masterAndZKLock) {
756         Exception exceptionCaught = null;
757         MasterProtocol master = null;
758         int tries = 0;
759         while (
760           !this.closed && master == null
761           ) {
762           tries++;
763           try {
764             master = createMasterInterface(masterProtocolState);
765           } catch (IOException e) {
766             exceptionCaught = e;
767           } catch (KeeperException e) {
768             exceptionCaught = e;
769           } catch (ServiceException e) {
770             exceptionCaught = e;
771           }
772 
773           if (exceptionCaught != null)
774             // It failed. If it's not the last try, we're going to wait a little
775           if (tries < numRetries) {
776             // tries at this point is 1 or more; decrement to start from 0.
777             long pauseTime = ConnectionUtils.getPauseTime(this.pause, tries - 1);
778             LOG.info("getMaster attempt " + tries + " of " + numRetries +
779               " failed; retrying after sleep of " +pauseTime + ", exception=" + exceptionCaught);
780 
781             try {
782               Thread.sleep(pauseTime);
783             } catch (InterruptedException e) {
784               Thread.currentThread().interrupt();
785               throw new RuntimeException(
786                 "Thread was interrupted while trying to connect to master.", e);
787             }
788 
789           } else {
790             // Enough tries, we stop now
791             LOG.info("getMaster attempt " + tries + " of " + numRetries +
792               " failed; no more retrying.", exceptionCaught);
793             throw new MasterNotRunningException(exceptionCaught);
794           }
795         }
796 
797         if (master == null) {
798           // implies this.closed true
799           throw new MasterNotRunningException(
800             "Connection was closed while trying to get master");
801         }
802 
803         return master;
804       }
805     }
806 
807     private void checkIfBaseNodeAvailable(ZooKeeperWatcher zkw)
808       throws MasterNotRunningException {
809       String errorMsg;
810       try {
811         if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {
812           errorMsg = "The node " + zkw.baseZNode+" is not in ZooKeeper. "
813             + "It should have been written by the master. "
814             + "Check the value configured in 'zookeeper.znode.parent'. "
815             + "There could be a mismatch with the one configured in the master.";
816           LOG.error(errorMsg);
817           throw new MasterNotRunningException(errorMsg);
818         }
819       } catch (KeeperException e) {
820         errorMsg = "Can't get connection to ZooKeeper: " + e.getMessage();
821         LOG.error(errorMsg);
822         throw new MasterNotRunningException(errorMsg, e);
823       }
824     }
825 
826     /**
827      * @return true if the master is running, throws an exception otherwise
828      * @throws MasterNotRunningException - if the master is not running
829      * @throws ZooKeeperConnectionException
830      */
831     @Override
832     public boolean isMasterRunning()
833       throws MasterNotRunningException, ZooKeeperConnectionException {
834       // When getting the master proxy connection, we check it's running,
835       // so if there is no exception, it means we've been able to get a
836       // connection on a running master
837       getKeepAliveMasterMonitor().close();
838       return true;
839     }
840 
841     @Override
842     public HRegionLocation getRegionLocation(final byte [] name,
843         final byte [] row, boolean reload)
844     throws IOException {
845       return reload? relocateRegion(name, row): locateRegion(name, row);
846     }
847 
848     @Override
849     public boolean isTableEnabled(byte[] tableName) throws IOException {
850       return testTableOnlineState(tableName, true);
851     }
852 
853     @Override
854     public boolean isTableDisabled(byte[] tableName) throws IOException {
855       return testTableOnlineState(tableName, false);
856     }
857 
858     @Override
859     public boolean isTableAvailable(final byte[] tableName) throws IOException {
860       final AtomicBoolean available = new AtomicBoolean(true);
861       final AtomicInteger regionCount = new AtomicInteger(0);
862       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
863         @Override
864         public boolean processRow(Result row) throws IOException {
865           HRegionInfo info = MetaScanner.getHRegionInfo(row);
866           if (info != null) {
867             if (Bytes.compareTo(tableName, info.getTableName()) == 0) {
868               ServerName server = HRegionInfo.getServerName(row);
869               if (server == null) {
870                 available.set(false);
871                 return false;
872               }
873               regionCount.incrementAndGet();
874             } else if (Bytes.compareTo(tableName, info.getTableName()) < 0) {
875               // Return if we are done with the current table
876               return false;
877             }
878           }
879           return true;
880         }
881       };
882       MetaScanner.metaScan(conf, visitor, tableName);
883       return available.get() && (regionCount.get() > 0);
884     }
885 
886     @Override
887     public boolean isTableAvailable(final byte[] tableName, final byte[][] splitKeys)
888         throws IOException {
889       final AtomicBoolean available = new AtomicBoolean(true);
890       final AtomicInteger regionCount = new AtomicInteger(0);
891       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
892         @Override
893         public boolean processRow(Result row) throws IOException {
894           HRegionInfo info = MetaScanner.getHRegionInfo(row);
895           if (info != null) {
896             if (Bytes.compareTo(tableName, info.getTableName()) == 0) {
897               ServerName server = HRegionInfo.getServerName(row);
898               if (server == null) {
899                 available.set(false);
900                 return false;
901               }
902               if (!Bytes.equals(info.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
903                 for (byte[] splitKey : splitKeys) {
904                   // Just check if the splitkey is available
905                   if (Bytes.equals(info.getStartKey(), splitKey)) {
906                     regionCount.incrementAndGet();
907                     break;
908                   }
909                 }
910               } else {
911                 // Always empty start row should be counted
912                 regionCount.incrementAndGet();
913               }
914             } else if (Bytes.compareTo(tableName, info.getTableName()) < 0) {
915               // Return if we are done with the current table
916               return false;
917             }
918           }
919           return true;
920         }
921       };
922       MetaScanner.metaScan(conf, visitor, tableName);
923       // +1 needs to be added so that the empty start row is also taken into account
924       return available.get() && (regionCount.get() == splitKeys.length + 1);
925     }
926 
927     /*
928      * @param enabled True if table is enabled
929      */
930     private boolean testTableOnlineState(byte [] tableName, boolean enabled)
931     throws IOException {
932       String tableNameStr = Bytes.toString(tableName);
933       ZooKeeperKeepAliveConnection zkw = getKeepAliveZooKeeperWatcher();
934       try {
935         if (enabled) {
936           return ZKTableReadOnly.isEnabledTable(zkw, tableNameStr);
937         }
938         return ZKTableReadOnly.isDisabledTable(zkw, tableNameStr);
939       } catch (KeeperException e) {
940         throw new IOException("Enable/Disable failed", e);
941       }finally {
942          zkw.close();
943       }
944     }
945 
946 
947     @Override
948     public HRegionLocation locateRegion(final byte[] regionName) throws IOException {
949       return locateRegion(HRegionInfo.getTableName(regionName),
950           HRegionInfo.getStartKey(regionName), false, true);
951     }
952 
953     @Override
954     public boolean isDeadServer(ServerName sn) {
955       if (clusterStatusListener == null) {
956         return false;
957       } else {
958         return clusterStatusListener.isDeadServer(sn);
959       }
960     }
961 
962     @Override
963     public List<HRegionLocation> locateRegions(final byte[] tableName)
964     throws IOException {
965       return locateRegions (tableName, false, true);
966     }
967 
968     @Override
969     public List<HRegionLocation> locateRegions(final byte[] tableName, final boolean useCache,
970         final boolean offlined) throws IOException {
971       NavigableMap<HRegionInfo, ServerName> regions = MetaScanner.allTableRegions(conf, tableName,
972         offlined);
973       final List<HRegionLocation> locations = new ArrayList<HRegionLocation>();
974       for (HRegionInfo regionInfo : regions.keySet()) {
975         locations.add(locateRegion(tableName, regionInfo.getStartKey(), useCache, true));
976       }
977       return locations;
978     }
979 
980     @Override
981     public HRegionLocation locateRegion(final byte [] tableName,
982         final byte [] row)
983     throws IOException{
984       return locateRegion(tableName, row, true, true);
985     }
986 
987     @Override
988     public HRegionLocation relocateRegion(final byte [] tableName,
989         final byte [] row)
990     throws IOException{
991 
992       // Since this is an explicit request not to use any caching, finding
993       // disabled tables should not be desirable.  This will ensure that an exception is thrown when
994       // the first time a disabled table is interacted with.
995       if (isTableDisabled(tableName)) {
996         throw new DoNotRetryIOException(Bytes.toString(tableName) + " is disabled.");
997       }
998 
999       return locateRegion(tableName, row, false, true);
1000     }
1001 
1002     private HRegionLocation locateRegion(final byte [] tableName,
1003       final byte [] row, boolean useCache, boolean retry)
1004     throws IOException {
1005       if (this.closed) throw new IOException(toString() + " closed");
1006       if (tableName == null || tableName.length == 0) {
1007         throw new IllegalArgumentException(
1008             "table name cannot be null or zero length");
1009       }
1010 
1011       if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
1012         ZooKeeperKeepAliveConnection zkw = getKeepAliveZooKeeperWatcher();
1013         try {
1014           if (LOG.isTraceEnabled()) {
1015             LOG.trace("Looking up meta region location in ZK," + " connection=" + this);
1016           }
1017           ServerName servername =
1018             MetaRegionTracker.blockUntilAvailable(zkw, this.rpcTimeout);
1019 
1020           if (LOG.isTraceEnabled()) {
1021             LOG.debug("Looked up meta region location, connection=" + this +
1022               "; serverName=" + ((servername == null) ? "null" : servername));
1023           }
1024           if (servername == null) return null;
1025           return new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, servername, 0);
1026         } catch (InterruptedException e) {
1027           Thread.currentThread().interrupt();
1028           return null;
1029         } finally {
1030           zkw.close();
1031         }
1032       } else {
1033         // Region not in the cache - have to go to the meta RS
1034         return locateRegionInMeta(HConstants.META_TABLE_NAME, tableName, row,
1035           useCache, userRegionLock, retry);
1036       }
1037     }
1038 
1039     /*
1040      * Search .META. for the HRegionLocation info that contains the table and
1041      * row we're seeking. It will prefetch certain number of regions info and
1042      * save them to the global region cache.
1043      */
1044     private void prefetchRegionCache(final byte[] tableName,
1045         final byte[] row) {
1046       // Implement a new visitor for MetaScanner, and use it to walk through
1047       // the .META.
1048       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
1049         public boolean processRow(Result result) throws IOException {
1050           try {
1051             HRegionInfo regionInfo = MetaScanner.getHRegionInfo(result);
1052             if (regionInfo == null) {
1053               return true;
1054             }
1055 
1056             // possible we got a region of a different table...
1057             if (!Bytes.equals(regionInfo.getTableName(), tableName)) {
1058               return false; // stop scanning
1059             }
1060             if (regionInfo.isOffline()) {
1061               // don't cache offline regions
1062               return true;
1063             }
1064 
1065             ServerName serverName = HRegionInfo.getServerName(result);
1066             if (serverName == null) {
1067               return true; // don't cache it
1068             }
1069             // instantiate the location
1070             long seqNum = HRegionInfo.getSeqNumDuringOpen(result);
1071             HRegionLocation loc = new HRegionLocation(regionInfo, serverName, seqNum);
1072             // cache this meta entry
1073             cacheLocation(tableName, null, loc);
1074             return true;
1075           } catch (RuntimeException e) {
1076             throw new IOException(e);
1077           }
1078         }
1079       };
1080       try {
1081         // pre-fetch certain number of regions info at region cache.
1082         MetaScanner.metaScan(conf, visitor, tableName, row,
1083             this.prefetchRegionLimit);
1084       } catch (IOException e) {
1085         LOG.warn("Encountered problems when prefetch META table: ", e);
1086       }
1087     }
1088 
1089     /*
1090       * Search the .META. table for the HRegionLocation
1091       * info that contains the table and row we're seeking.
1092       */
1093     private HRegionLocation locateRegionInMeta(final byte [] parentTable,
1094       final byte [] tableName, final byte [] row, boolean useCache,
1095       Object regionLockObject, boolean retry)
1096     throws IOException {
1097       HRegionLocation location;
1098       // If we are supposed to be using the cache, look in the cache to see if
1099       // we already have the region.
1100       if (useCache) {
1101         location = getCachedLocation(tableName, row);
1102         if (location != null) {
1103           return location;
1104         }
1105       }
1106       int localNumRetries = retry ? numRetries : 1;
1107       // build the key of the meta region we should be looking for.
1108       // the extra 9's on the end are necessary to allow "exact" matches
1109       // without knowing the precise region names.
1110       byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
1111         HConstants.NINES, false);
1112       for (int tries = 0; true; tries++) {
1113         if (tries >= localNumRetries) {
1114           throw new NoServerForRegionException("Unable to find region for "
1115             + Bytes.toStringBinary(row) + " after " + numRetries + " tries.");
1116         }
1117 
1118         HRegionLocation metaLocation = null;
1119         try {
1120           // locate the meta region
1121           metaLocation = locateRegion(parentTable, metaKey, true, false);
1122           // If null still, go around again.
1123           if (metaLocation == null) continue;
1124           ClientProtocol server = getClient(metaLocation.getServerName());
1125 
1126           Result regionInfoRow;
1127           // This block guards against two threads trying to load the meta
1128           // region at the same time. The first will load the meta region and
1129           // the second will use the value that the first one found.
1130           synchronized (regionLockObject) {
1131             // If the parent table is META, we may want to pre-fetch some
1132             // region info into the global region cache for this table.
1133             if (Bytes.equals(parentTable, HConstants.META_TABLE_NAME) &&
1134                 (getRegionCachePrefetch(tableName)) )  {
1135               prefetchRegionCache(tableName, row);
1136             }
1137 
1138             // Check the cache again for a hit in case some other thread made the
1139             // same query while we were waiting on the lock. If not supposed to
1140             // be using the cache, delete any existing cached location so it won't
1141             // interfere.
1142             if (useCache) {
1143               location = getCachedLocation(tableName, row);
1144               if (location != null) {
1145                 return location;
1146               }
1147             } else {
1148               forceDeleteCachedLocation(tableName, row);
1149             }
1150 
1151             // Query the meta region for the location of the meta region
1152             regionInfoRow = ProtobufUtil.getRowOrBefore(server,
1153               metaLocation.getRegionInfo().getRegionName(), metaKey,
1154               HConstants.CATALOG_FAMILY);
1155           }
1156           if (regionInfoRow == null) {
1157             throw new TableNotFoundException(Bytes.toString(tableName));
1158           }
1159 
1160           // convert the row result into the HRegionLocation we need!
1161           HRegionInfo regionInfo = MetaScanner.getHRegionInfo(regionInfoRow);
1162           if (regionInfo == null) {
1163             throw new IOException("HRegionInfo was null or empty in " +
1164               Bytes.toString(parentTable) + ", row=" + regionInfoRow);
1165           }
1166 
1167           // possible we got a region of a different table...
1168           if (!Bytes.equals(regionInfo.getTableName(), tableName)) {
1169             throw new TableNotFoundException(
1170                   "Table '" + Bytes.toString(tableName) + "' was not found, got: " +
1171                   Bytes.toString(regionInfo.getTableName()) + ".");
1172           }
1173           if (regionInfo.isSplit()) {
1174             throw new RegionOfflineException("the only available region for" +
1175               " the required row is a split parent," +
1176               " the daughters should be online soon: " +
1177               regionInfo.getRegionNameAsString());
1178           }
1179           if (regionInfo.isOffline()) {
1180             throw new RegionOfflineException("the region is offline, could" +
1181               " be caused by a disable table call: " +
1182               regionInfo.getRegionNameAsString());
1183           }
1184 
1185           ServerName serverName = HRegionInfo.getServerName(regionInfoRow);
1186           if (serverName == null) {
1187             throw new NoServerForRegionException("No server address listed " +
1188               "in " + Bytes.toString(parentTable) + " for region " +
1189               regionInfo.getRegionNameAsString() + " containing row " +
1190               Bytes.toStringBinary(row));
1191           }
1192 
1193           if (isDeadServer(serverName)){
1194             throw new RegionServerStoppedException(".META. says the region "+
1195                 regionInfo.getRegionNameAsString()+" is managed by the server " + serverName +
1196                 ", but it is dead.");
1197           }
1198 
1199           // Instantiate the location
1200           location = new HRegionLocation(regionInfo, serverName,
1201             HRegionInfo.getSeqNumDuringOpen(regionInfoRow));
1202           cacheLocation(tableName, null, location);
1203           return location;
1204         } catch (TableNotFoundException e) {
1205           // if we got this error, probably means the table just plain doesn't
1206           // exist. rethrow the error immediately. this should always be coming
1207           // from the HTable constructor.
1208           throw e;
1209         } catch (IOException e) {
1210           if (e instanceof RemoteException) {
1211             e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
1212           }
1213           if (tries < numRetries - 1) {
1214             if (LOG.isDebugEnabled()) {
1215               LOG.debug("locateRegionInMeta parentTable=" +
1216                 Bytes.toString(parentTable) + ", metaLocation=" +
1217                 ((metaLocation == null)? "null": "{" + metaLocation + "}") +
1218                 ", attempt=" + tries + " of " +
1219                 this.numRetries + " failed; retrying after sleep of " +
1220                 ConnectionUtils.getPauseTime(this.pause, tries) + " because: " + e.getMessage());
1221             }
1222           } else {
1223             throw e;
1224           }
1225           // Only relocate the parent region if necessary
1226           if(!(e instanceof RegionOfflineException ||
1227               e instanceof NoServerForRegionException)) {
1228             relocateRegion(parentTable, metaKey);
1229           }
1230         }
1231         try{
1232           Thread.sleep(ConnectionUtils.getPauseTime(this.pause, tries));
1233         } catch (InterruptedException e) {
1234           Thread.currentThread().interrupt();
1235           throw new IOException("Giving up trying to location region in " +
1236             "meta: thread is interrupted.");
1237         }
1238       }
1239     }
1240 
1241     /*
1242      * Search the cache for a location that fits our table and row key.
1243      * Return null if no suitable region is located. TODO: synchronization note
1244      *
1245      * <p>TODO: This method during writing consumes 15% of CPU doing lookup
1246      * into the Soft Reference SortedMap.  Improve.
1247      *
1248      * @param tableName
1249      * @param row
1250      * @return Null or region location found in cache.
1251      */
1252     HRegionLocation getCachedLocation(final byte [] tableName,
1253         final byte [] row) {
1254       SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
1255         getTableLocations(tableName);
1256 
1257       // start to examine the cache. we can only do cache actions
1258       // if there's something in the cache for this table.
1259       if (tableLocations.isEmpty()) {
1260         return null;
1261       }
1262 
1263       HRegionLocation possibleRegion = tableLocations.get(row);
1264       if (possibleRegion != null) {
1265         return possibleRegion;
1266       }
1267 
1268       possibleRegion = tableLocations.lowerValueByKey(row);
1269       if (possibleRegion == null) {
1270         return null;
1271       }
1272 
1273       // make sure that the end key is greater than the row we're looking
1274       // for, otherwise the row actually belongs in the next region, not
1275       // this one. the exception case is when the endkey is
1276       // HConstants.EMPTY_END_ROW, signifying that the region we're
1277       // checking is actually the last region in the table.
1278       byte[] endKey = possibleRegion.getRegionInfo().getEndKey();
1279       if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
1280           KeyValue.getRowComparator(tableName).compareRows(
1281               endKey, 0, endKey.length, row, 0, row.length) > 0) {
1282         return possibleRegion;
1283       }
1284 
1285       // Passed all the way through, so we got nothing - complete cache miss
1286       return null;
1287     }
1288 
1289     /**
1290      * Delete a cached location, no matter what it is. Called when we were told to not use cache.
1291      * @param tableName tableName
1292      * @param row
1293      */
1294     void forceDeleteCachedLocation(final byte [] tableName, final byte [] row) {
1295       HRegionLocation rl = null;
1296       synchronized (this.cachedRegionLocations) {
1297         Map<byte[], HRegionLocation> tableLocations = getTableLocations(tableName);
1298         // start to examine the cache. we can only do cache actions
1299         // if there's something in the cache for this table.
1300         if (!tableLocations.isEmpty()) {
1301           rl = getCachedLocation(tableName, row);
1302           if (rl != null) {
1303             tableLocations.remove(rl.getRegionInfo().getStartKey());
1304           }
1305         }
1306       }
1307       if ((rl != null) && LOG.isDebugEnabled()) {
1308         LOG.debug("Removed " + rl.getHostname() + ":" + rl.getPort()
1309           + " as a location of " + rl.getRegionInfo().getRegionNameAsString() +
1310           " for tableName=" + Bytes.toString(tableName) + " from cache");
1311       }
1312     }
1313 
1314     /*
1315      * Delete all cached entries of a table that maps to a specific location.
1316      */
1317     @Override
1318     public void clearCaches(final ServerName serverName){
1319       boolean deletedSomething = false;
1320       synchronized (this.cachedRegionLocations) {
1321         if (!cachedServers.contains(serverName)) {
1322           return;
1323         }
1324         for (Map<byte[], HRegionLocation> tableLocations :
1325             cachedRegionLocations.values()) {
1326           for (Entry<byte[], HRegionLocation> e : tableLocations.entrySet()) {
1327             if (serverName.equals(e.getValue().getServerName())) {
1328               tableLocations.remove(e.getKey());
1329               deletedSomething = true;
1330             }
1331           }
1332         }
1333         cachedServers.remove(serverName);
1334       }
1335       if (deletedSomething && LOG.isDebugEnabled()) {
1336         LOG.debug("Removed all cached region locations that map to " + serverName);
1337       }
1338     }
1339 
1340     /*
1341      * @param tableName
1342      * @return Map of cached locations for passed <code>tableName</code>
1343      */
1344     private SoftValueSortedMap<byte [], HRegionLocation> getTableLocations(
1345         final byte [] tableName) {
1346       // find the map of cached locations for this table
1347       Integer key = Bytes.mapKey(tableName);
1348       SoftValueSortedMap<byte [], HRegionLocation> result;
1349       synchronized (this.cachedRegionLocations) {
1350         result = this.cachedRegionLocations.get(key);
1351         // if tableLocations for this table isn't built yet, make one
1352         if (result == null) {
1353           result = new SoftValueSortedMap<byte [], HRegionLocation>(
1354               Bytes.BYTES_COMPARATOR);
1355           this.cachedRegionLocations.put(key, result);
1356         }
1357       }
1358       return result;
1359     }
1360 
1361     @Override
1362     public void clearRegionCache() {
1363       synchronized(this.cachedRegionLocations) {
1364         this.cachedRegionLocations.clear();
1365         this.cachedServers.clear();
1366       }
1367     }
1368 
1369     @Override
1370     public void clearRegionCache(final byte [] tableName) {
1371       synchronized (this.cachedRegionLocations) {
1372         this.cachedRegionLocations.remove(Bytes.mapKey(tableName));
1373       }
1374     }
1375 
1376     /**
1377      * Put a newly discovered HRegionLocation into the cache.
1378      * @param tableName The table name.
1379      * @param source the source of the new location, if it's not coming from meta
1380      * @param location the new location
1381      */
1382     private void cacheLocation(final byte [] tableName, final HRegionLocation source,
1383         final HRegionLocation location) {
1384       boolean isFromMeta = (source == null);
1385       byte [] startKey = location.getRegionInfo().getStartKey();
1386       Map<byte [], HRegionLocation> tableLocations =
1387         getTableLocations(tableName);
1388       boolean isNewCacheEntry = false;
1389       boolean isStaleUpdate = false;
1390       HRegionLocation oldLocation = null;
1391       synchronized (this.cachedRegionLocations) {
1392         cachedServers.add(location.getServerName());
1393         oldLocation = tableLocations.get(startKey);
1394         isNewCacheEntry = (oldLocation == null);
1395         // If the server in cache sends us a redirect, assume it's always valid.
1396         if (!isNewCacheEntry && !oldLocation.equals(source)) {
1397           long newLocationSeqNum = location.getSeqNum();
1398           // Meta record is stale - some (probably the same) server has closed the region
1399           // with later seqNum and told us about the new location.
1400           boolean isStaleMetaRecord = isFromMeta && (oldLocation.getSeqNum() > newLocationSeqNum);
1401           // Same as above for redirect. However, in this case, if the number is equal to previous
1402           // record, the most common case is that first the region was closed with seqNum, and then
1403           // opened with the same seqNum; hence we will ignore the redirect.
1404           // There are so many corner cases with various combinations of opens and closes that
1405           // an additional counter on top of seqNum would be necessary to handle them all.
1406           boolean isStaleRedirect = !isFromMeta && (oldLocation.getSeqNum() >= newLocationSeqNum);
1407           isStaleUpdate = (isStaleMetaRecord || isStaleRedirect);
1408         }
1409         if (!isStaleUpdate) {
1410           tableLocations.put(startKey, location);
1411         }
1412       }
1413       if (isNewCacheEntry) {
1414         LOG.debug("Cached location for " +
1415             location.getRegionInfo().getRegionNameAsString() +
1416             " is " + location.getHostnamePort());
1417       } else if (isStaleUpdate && !location.equals(oldLocation)) {
1418         LOG.debug("Ignoring stale location update for "
1419           + location.getRegionInfo().getRegionNameAsString() + ": "
1420           + location.getHostnamePort() + " at " + location.getSeqNum() + "; local "
1421           + oldLocation.getHostnamePort() + " at " + oldLocation.getSeqNum());
1422       }
1423     }
1424 
1425     @Override
1426     @Deprecated
1427     public AdminProtocol getAdmin(final String hostname, final int port) throws IOException {
1428       return getAdmin(new ServerName(hostname, port, 0L));
1429     }
1430 
1431     @Override
1432     public AdminProtocol getAdmin(final ServerName serverName)
1433         throws IOException {
1434       return getAdmin(serverName, false);
1435     }
1436 
1437     @Override
1438     @Deprecated
1439     public ClientProtocol getClient(final String hostname, final int port)
1440     throws IOException {
1441       return (ClientProtocol)getProtocol(hostname, port, clientClass);
1442     }
1443 
1444     @Override
1445     public ClientProtocol getClient(final ServerName serverName)
1446         throws IOException {
1447       if (isDeadServer(serverName)){
1448         throw new RegionServerStoppedException("The server " + serverName + " is dead.");
1449       }
1450       return (ClientProtocol)
1451           getProtocol(serverName.getHostname(), serverName.getPort(), clientClass);
1452     }
1453 
1454     @Override
1455     @Deprecated
1456     public AdminProtocol getAdmin(final String hostname, final int port,
1457         final boolean master)
1458     throws IOException {
1459       return (AdminProtocol)getProtocol(hostname, port, adminClass);
1460     }
1461 
1462     @Override
1463     public AdminProtocol getAdmin(final ServerName serverName, final boolean master)
1464         throws IOException {
1465       if (isDeadServer(serverName)){
1466         throw new RegionServerStoppedException("The server " + serverName + " is dead.");
1467       }
1468       return (AdminProtocol)getProtocol(
1469           serverName.getHostname(), serverName.getPort(), adminClass);
1470     }
1471 
1472     /**
1473      * Either the passed <code>isa</code> is null or <code>hostname</code>
1474      * can be but not both.
1475      * @param hostname
1476      * @param port
1477      * @param protocolClass
1478      * @return Proxy.
1479      * @throws IOException
1480      */
1481     IpcProtocol getProtocol(final String hostname,
1482         final int port, final Class <? extends IpcProtocol> protocolClass)
1483     throws IOException {
1484       String rsName = Addressing.createHostAndPortStr(hostname, port);
1485       // See if we already have a connection (common case)
1486       Map<String, IpcProtocol> protocols = this.servers.get(rsName);
1487       if (protocols == null) {
1488         protocols = new HashMap<String, IpcProtocol>();
1489         Map<String, IpcProtocol> existingProtocols =
1490           this.servers.putIfAbsent(rsName, protocols);
1491         if (existingProtocols != null) {
1492           protocols = existingProtocols;
1493         }
1494       }
1495       String protocol = protocolClass.getName();
1496       IpcProtocol server = protocols.get(protocol);
1497       if (server == null) {
1498         // create a unique lock for this RS + protocol (if necessary)
1499         String lockKey = protocol + "@" + rsName;
1500         this.connectionLock.putIfAbsent(lockKey, lockKey);
1501         // get the RS lock
1502         synchronized (this.connectionLock.get(lockKey)) {
1503           // do one more lookup in case we were stalled above
1504           server = protocols.get(protocol);
1505           if (server == null) {
1506             try {
1507               // Only create isa when we need to.
1508               InetSocketAddress address = new InetSocketAddress(hostname, port);
1509               // definitely a cache miss. establish an RPC for this RS
1510               server = HBaseClientRPC.waitForProxy(rpcEngine, protocolClass, address, this.conf,
1511                   this.maxRPCAttempts, this.rpcTimeout, this.rpcTimeout);
1512               protocols.put(protocol, server);
1513             } catch (RemoteException e) {
1514               LOG.warn("RemoteException connecting to RS", e);
1515               // Throw what the RemoteException was carrying.
1516               throw e.unwrapRemoteException();
1517             }
1518           }
1519         }
1520       }
1521       return server;
1522     }
1523 
1524     @Override
1525     @Deprecated
1526     public ZooKeeperWatcher getZooKeeperWatcher()
1527         throws ZooKeeperConnectionException {
1528       canCloseZKW = false;
1529 
1530       try {
1531         return getKeepAliveZooKeeperWatcher();
1532       } catch (ZooKeeperConnectionException e){
1533         throw e;
1534       }catch (IOException e) {
1535         // Encapsulate exception to keep interface
1536         throw new ZooKeeperConnectionException(
1537           "Can't create a zookeeper connection", e);
1538       }
1539     }
1540 
1541 
1542     private ZooKeeperKeepAliveConnection keepAliveZookeeper;
1543     private int keepAliveZookeeperUserCount;
1544     private boolean canCloseZKW = true;
1545 
1546     // keepAlive time, in ms. No reason to make it configurable.
1547     private static final long keepAlive = 5 * 60 * 1000;
1548 
1549     /**
1550      * Retrieve a shared ZooKeeperWatcher. You must close it it once you've have
1551      *  finished with it.
1552      * @return The shared instance. Never returns null.
1553      */
1554     public ZooKeeperKeepAliveConnection getKeepAliveZooKeeperWatcher()
1555       throws IOException {
1556       synchronized (masterAndZKLock) {
1557 
1558         if (keepAliveZookeeper == null) {
1559           // We don't check that our link to ZooKeeper is still valid
1560           // But there is a retry mechanism in the ZooKeeperWatcher itself
1561           keepAliveZookeeper = new ZooKeeperKeepAliveConnection(
1562             conf, this.toString(), this);
1563         }
1564         keepAliveZookeeperUserCount++;
1565         keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1566 
1567         return keepAliveZookeeper;
1568       }
1569     }
1570 
1571     void releaseZooKeeperWatcher(ZooKeeperWatcher zkw) {
1572       if (zkw == null){
1573         return;
1574       }
1575       synchronized (masterAndZKLock) {
1576         --keepAliveZookeeperUserCount;
1577         if (keepAliveZookeeperUserCount <=0 ){
1578           keepZooKeeperWatcherAliveUntil =
1579             System.currentTimeMillis() + keepAlive;
1580         }
1581       }
1582     }
1583 
1584 
1585     /**
1586      * Creates a Chore thread to check the connections to master & zookeeper
1587      *  and close them when they reach their closing time (
1588      *  {@link MasterProtocolState#keepAliveUntil} and
1589      *  {@link #keepZooKeeperWatcherAliveUntil}). Keep alive time is
1590      *  managed by the release functions and the variable {@link #keepAlive}
1591      */
1592     private static class DelayedClosing extends Chore implements Stoppable {
1593       private HConnectionImplementation hci;
1594       Stoppable stoppable;
1595 
1596       private DelayedClosing(
1597         HConnectionImplementation hci, Stoppable stoppable){
1598         super(
1599           "ZooKeeperWatcher and Master delayed closing for connection "+hci,
1600           60*1000, // We check every minutes
1601           stoppable);
1602         this.hci = hci;
1603         this.stoppable = stoppable;
1604       }
1605 
1606       static DelayedClosing createAndStart(HConnectionImplementation hci){
1607         Stoppable stoppable = new Stoppable() {
1608               private volatile boolean isStopped = false;
1609               @Override public void stop(String why) { isStopped = true;}
1610               @Override public boolean isStopped() {return isStopped;}
1611             };
1612 
1613         return new DelayedClosing(hci, stoppable);
1614       }
1615 
1616       protected void closeMasterProtocol(MasterProtocolState protocolState) {
1617         if (System.currentTimeMillis() > protocolState.keepAliveUntil) {
1618           hci.closeMasterProtocol(protocolState);
1619           protocolState.keepAliveUntil = Long.MAX_VALUE;
1620         }
1621       }
1622 
1623       @Override
1624       protected void chore() {
1625         synchronized (hci.masterAndZKLock) {
1626           if (hci.canCloseZKW) {
1627             if (System.currentTimeMillis() >
1628               hci.keepZooKeeperWatcherAliveUntil) {
1629 
1630               hci.closeZooKeeperWatcher();
1631               hci.keepZooKeeperWatcherAliveUntil = Long.MAX_VALUE;
1632             }
1633           }
1634           closeMasterProtocol(hci.masterAdminProtocol);
1635           closeMasterProtocol(hci.masterMonitorProtocol);
1636         }
1637       }
1638 
1639       @Override
1640       public void stop(String why) {
1641         stoppable.stop(why);
1642       }
1643 
1644       @Override
1645       public boolean isStopped() {
1646         return stoppable.isStopped();
1647       }
1648     }
1649 
1650     private void closeZooKeeperWatcher() {
1651       synchronized (masterAndZKLock) {
1652         if (keepAliveZookeeper != null) {
1653           LOG.info("Closing zookeeper sessionid=0x" +
1654             Long.toHexString(
1655               keepAliveZookeeper.getRecoverableZooKeeper().getSessionId()));
1656           keepAliveZookeeper.internalClose();
1657           keepAliveZookeeper = null;
1658         }
1659         keepAliveZookeeperUserCount = 0;
1660       }
1661     }
1662 
1663     private static class MasterProtocolHandler implements InvocationHandler {
1664       private HConnectionImplementation connection;
1665       private MasterProtocolState protocolStateTracker;
1666 
1667       protected MasterProtocolHandler(HConnectionImplementation connection,
1668                                     MasterProtocolState protocolStateTracker) {
1669         this.connection = connection;
1670         this.protocolStateTracker = protocolStateTracker;
1671       }
1672 
1673       @Override
1674       public Object invoke(Object proxy, Method method, Object[] args)
1675         throws Throwable {
1676         if (method.getName().equals("close") &&
1677               method.getParameterTypes().length == 0) {
1678           release(connection, protocolStateTracker);
1679           return null;
1680         } else {
1681           try {
1682             return method.invoke(protocolStateTracker.protocol, args);
1683           }catch  (InvocationTargetException e){
1684             // We will have this for all the exception, checked on not, sent
1685             //  by any layer, including the functional exception
1686             Throwable cause = e.getCause();
1687             if (cause == null){
1688               throw new RuntimeException(
1689                 "Proxy invocation failed and getCause is null", e);
1690             }
1691             if (cause instanceof UndeclaredThrowableException) {
1692               cause = cause.getCause();
1693             }
1694             throw cause;
1695           }
1696         }
1697       }
1698 
1699       private void release(
1700         HConnectionImplementation connection,
1701         MasterProtocolState target) {
1702         connection.releaseMaster(target);
1703       }
1704     }
1705 
1706     MasterProtocolState masterAdminProtocol =
1707       new MasterProtocolState(MasterAdminProtocol.class);
1708     MasterProtocolState masterMonitorProtocol =
1709       new MasterProtocolState(MasterMonitorProtocol.class);
1710 
1711     /**
1712      * This function allows HBaseAdmin and potentially others
1713      * to get a shared master connection.
1714      *
1715      * @return The shared instance. Never returns null.
1716      * @throws MasterNotRunningException
1717      */
1718     private Object getKeepAliveMasterProtocol(
1719         MasterProtocolState protocolState, Class connectionClass)
1720         throws MasterNotRunningException {
1721       synchronized (masterAndZKLock) {
1722         if (!isKeepAliveMasterConnectedAndRunning(protocolState)) {
1723           protocolState.protocol = null;
1724           protocolState.protocol = createMasterWithRetries(protocolState);
1725         }
1726         protocolState.userCount++;
1727         protocolState.keepAliveUntil = Long.MAX_VALUE;
1728 
1729         return Proxy.newProxyInstance(
1730           connectionClass.getClassLoader(),
1731           new Class[]{connectionClass},
1732           new MasterProtocolHandler(this, protocolState)
1733         );
1734       }
1735     }
1736 
1737     @Override
1738     public MasterAdminProtocol getMasterAdmin() throws MasterNotRunningException {
1739       return getKeepAliveMasterAdmin();
1740     }
1741 
1742     @Override
1743     public MasterMonitorProtocol getMasterMonitor() throws MasterNotRunningException {
1744       return getKeepAliveMasterMonitor();
1745     }
1746 
1747     @Override
1748     public MasterAdminKeepAliveConnection getKeepAliveMasterAdmin()
1749         throws MasterNotRunningException {
1750       return (MasterAdminKeepAliveConnection)
1751         getKeepAliveMasterProtocol(masterAdminProtocol, MasterAdminKeepAliveConnection.class);
1752     }
1753 
1754     @Override
1755     public MasterMonitorKeepAliveConnection getKeepAliveMasterMonitor()
1756         throws MasterNotRunningException {
1757       return (MasterMonitorKeepAliveConnection)
1758         getKeepAliveMasterProtocol(masterMonitorProtocol, MasterMonitorKeepAliveConnection.class);
1759     }
1760 
1761     private boolean isKeepAliveMasterConnectedAndRunning(MasterProtocolState protocolState){
1762       if (protocolState.protocol == null){
1763         return false;
1764       }
1765       try {
1766          return protocolState.protocol.isMasterRunning(
1767            null, RequestConverter.buildIsMasterRunningRequest()).getIsMasterRunning();
1768       }catch (UndeclaredThrowableException e){
1769         // It's somehow messy, but we can receive exceptions such as
1770         //  java.net.ConnectException but they're not declared. So we catch
1771         //  it...
1772         LOG.info("Master connection is not running anymore",
1773           e.getUndeclaredThrowable());
1774         return false;
1775       } catch (ServiceException se) {
1776         LOG.warn("Checking master connection", se);
1777         return false;
1778       }
1779     }
1780 
1781    private void releaseMaster(MasterProtocolState protocolState) {
1782       if (protocolState.protocol == null){
1783         return;
1784       }
1785       synchronized (masterAndZKLock) {
1786         --protocolState.userCount;
1787         if (protocolState.userCount <= 0) {
1788           protocolState.keepAliveUntil =
1789             System.currentTimeMillis() + keepAlive;
1790         }
1791       }
1792     }
1793 
1794     private void closeMasterProtocol(MasterProtocolState protocolState) {
1795       if (protocolState.protocol != null){
1796         LOG.info("Closing master protocol: " + protocolState.protocolClass.getName());
1797         protocolState.protocol = null;
1798       }
1799       protocolState.userCount = 0;
1800     }
1801 
1802     /**
1803      * Immediate close of the shared master. Can be by the delayed close or
1804      *  when closing the connection itself.
1805      */
1806     private void closeMaster() {
1807       synchronized (masterAndZKLock) {
1808         closeMasterProtocol(masterAdminProtocol);
1809         closeMasterProtocol(masterMonitorProtocol);
1810       }
1811     }
1812 
1813     @Override
1814     public <T> T getRegionServerWithRetries(ServerCallable<T> callable)
1815     throws IOException, RuntimeException {
1816       return callable.withRetries();
1817     }
1818 
1819     @Override
1820     public <T> T getRegionServerWithoutRetries(ServerCallable<T> callable)
1821     throws IOException, RuntimeException {
1822       return callable.withoutRetries();
1823     }
1824 
1825     @Deprecated
1826     private <R> Callable<MultiResponse> createCallable(final HRegionLocation loc,
1827         final MultiAction<R> multi, final byte[] tableName) {
1828       // TODO: This does not belong in here!!! St.Ack HConnections should
1829       // not be dealing in Callables; Callables have HConnections, not other
1830       // way around.
1831       final HConnection connection = this;
1832       return new Callable<MultiResponse>() {
1833         @Override
1834         public MultiResponse call() throws Exception {
1835           ServerCallable<MultiResponse> callable =
1836             new MultiServerCallable<R>(connection, tableName, loc, multi);
1837           return callable.withoutRetries();
1838         }
1839       };
1840    }
1841 
1842    void updateCachedLocation(HRegionInfo hri, HRegionLocation source,
1843        ServerName serverName, long seqNum) {
1844       HRegionLocation newHrl = new HRegionLocation(hri, serverName, seqNum);
1845       synchronized (this.cachedRegionLocations) {
1846         cacheLocation(hri.getTableName(), source, newHrl);
1847       }
1848     }
1849 
1850    /**
1851     * Deletes the cached location of the region if necessary, based on some error from source.
1852     * @param hri The region in question.
1853     * @param source The source of the error that prompts us to invalidate cache.
1854     */
1855     void deleteCachedLocation(HRegionInfo hri, HRegionLocation source) {
1856       boolean isStaleDelete = false;
1857       HRegionLocation oldLocation = null;
1858       synchronized (this.cachedRegionLocations) {
1859         Map<byte[], HRegionLocation> tableLocations =
1860           getTableLocations(hri.getTableName());
1861         oldLocation = tableLocations.get(hri.getStartKey());
1862         if (oldLocation != null) {
1863            // Do not delete the cache entry if it's not for the same server that gave us the error.
1864           isStaleDelete = (source != null) && !oldLocation.equals(source);
1865           if (!isStaleDelete) {
1866             tableLocations.remove(hri.getStartKey());
1867           }
1868         }
1869       }
1870       if (isStaleDelete) {
1871         LOG.debug("Received an error from " + source.getHostnamePort() + " for region "
1872           + hri.getRegionNameAsString() + "; not removing "
1873           + oldLocation.getHostnamePort() + " from cache.");
1874       }
1875     }
1876 
1877     /**
1878      * Update the location with the new value (if the exception is a RegionMovedException)
1879      * or delete it from the cache.
1880      * @param exception an object (to simplify user code) on which we will try to find a nested
1881      *                  or wrapped or both RegionMovedException
1882      * @param source server that is the source of the location update.
1883      */
1884     private void updateCachedLocations(final byte[] tableName, Row row,
1885       final Object exception, final HRegionLocation source) {
1886       if (row == null || tableName == null) {
1887         LOG.warn("Coding error, see method javadoc. row=" + (row == null ? "null" : row) +
1888             ", tableName=" + (tableName == null ? "null" : Bytes.toString(tableName)));
1889         return;
1890       }
1891 
1892       // Is it something we have already updated?
1893       final HRegionLocation oldLocation = getCachedLocation(tableName, row.getRow());
1894       if (oldLocation == null) {
1895         // There is no such location in the cache => it's been removed already => nothing to do
1896         return;
1897       }
1898 
1899       HRegionInfo regionInfo = oldLocation.getRegionInfo();
1900       final RegionMovedException rme = RegionMovedException.find(exception);
1901       if (rme != null) {
1902         LOG.info("Region " + regionInfo.getRegionNameAsString() + " moved to " +
1903           rme.getHostname() + ":" + rme.getPort() + " according to " + source.getHostnamePort());
1904         updateCachedLocation(
1905             regionInfo, source, rme.getServerName(), rme.getLocationSeqNum());
1906       } else if (RegionOpeningException.find(exception) != null) {
1907         LOG.info("Region " + regionInfo.getRegionNameAsString() + " is being opened on "
1908           + source.getHostnamePort() + "; not deleting the cache entry");
1909       } else {
1910         deleteCachedLocation(regionInfo, source);
1911       }
1912     }
1913 
1914     @Override
1915     @Deprecated
1916     public void processBatch(List<? extends Row> list,
1917         final byte[] tableName,
1918         ExecutorService pool,
1919         Object[] results) throws IOException, InterruptedException {
1920       // This belongs in HTable!!! Not in here.  St.Ack
1921 
1922       // results must be the same size as list
1923       if (results.length != list.size()) {
1924         throw new IllegalArgumentException(
1925           "argument results must be the same size as argument list");
1926       }
1927       processBatchCallback(list, tableName, pool, results, null);
1928     }
1929 
1930     /**
1931      * Send the queries in parallel on the different region servers. Retries on failures.
1932      * If the method returns it means that there is no error, and the 'results' array will
1933      * contain no exception. On error, an exception is thrown, and the 'results' array will
1934      * contain results and exceptions.
1935      * @deprecated since 0.96 - Use {@link HTable#processBatchCallback} instead
1936      */
1937     @Override
1938     @Deprecated
1939     public <R> void processBatchCallback(
1940       List<? extends Row> list,
1941       byte[] tableName,
1942       ExecutorService pool,
1943       Object[] results,
1944       Batch.Callback<R> callback)
1945       throws IOException, InterruptedException {
1946 
1947       Process<R> p = new Process<R>(this, list, tableName, pool, results, callback);
1948       p.processBatchCallback();
1949     }
1950 
1951 
1952     /**
1953      * Methods and attributes to manage a batch process are grouped into this single class.
1954      * This allows, by creating a Process<R> per batch process to ensure multithread safety.
1955      *
1956      * This code should be move to HTable once processBatchCallback is not supported anymore in
1957      * the HConnection interface.
1958      */
1959     private static class Process<R> {
1960       // Info on the queries and their context
1961       private final HConnectionImplementation hci;
1962       private final List<? extends Row> rows;
1963       private final byte[] tableName;
1964       private final ExecutorService pool;
1965       private final Object[] results;
1966       private final Batch.Callback<R> callback;
1967 
1968       // Used during the batch process
1969       private final List<Action<R>> toReplay;
1970       private final LinkedList<Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>>>
1971         inProgress;
1972       private int curNumRetries;
1973 
1974       // Notified when a tasks is done
1975       private final List<MultiAction<R>> finishedTasks = new ArrayList<MultiAction<R>>();
1976 
1977       private Process(HConnectionImplementation hci, List<? extends Row> list,
1978                        byte[] tableName, ExecutorService pool, Object[] results,
1979                        Batch.Callback<R> callback){
1980         this.hci = hci;
1981         this.rows = list;
1982         this.tableName = tableName;
1983         this.pool = pool;
1984         this.results = results;
1985         this.callback = callback;
1986         this.toReplay = new ArrayList<Action<R>>();
1987         this.inProgress =
1988           new LinkedList<Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>>>();
1989         this.curNumRetries = 0;
1990       }
1991 
1992 
1993       /**
1994        * Group a list of actions per region servers, and send them. The created MultiActions are
1995        *  added to the inProgress list.
1996        * @param actionsList
1997        * @param sleepTime - sleep time before actually executing the actions. Can be zero.
1998        * @throws IOException - if we can't locate a region after multiple retries.
1999        */
2000       private void submit(List<Action<R>> actionsList, final long sleepTime) throws IOException {
2001         // group per location => regions server
2002         final Map<HRegionLocation, MultiAction<R>> actionsByServer =
2003           new HashMap<HRegionLocation, MultiAction<R>>();
2004         for (Action<R> aAction : actionsList) {
2005           final Row row = aAction.getAction();
2006 
2007           if (row != null) {
2008             final HRegionLocation loc = hci.locateRegion(this.tableName, row.getRow());
2009             if (loc == null) {
2010               throw new IOException("No location found, aborting submit.");
2011             }
2012 
2013             final byte[] regionName = loc.getRegionInfo().getRegionName();
2014             MultiAction<R> actions = actionsByServer.get(loc);
2015             if (actions == null) {
2016               actions = new MultiAction<R>();
2017               actionsByServer.put(loc, actions);
2018             }
2019             actions.add(regionName, aAction);
2020           }
2021         }
2022 
2023         // Send the queries and add them to the inProgress list
2024         for (Entry<HRegionLocation, MultiAction<R>> e : actionsByServer.entrySet()) {
2025           Callable<MultiResponse> callable =
2026             createDelayedCallable(sleepTime, e.getKey(), e.getValue());
2027           if (LOG.isTraceEnabled() && (sleepTime > 0)) {
2028             StringBuilder sb = new StringBuilder();
2029             for (Action<R> action : e.getValue().allActions()) {
2030               sb.append(Bytes.toStringBinary(action.getAction().getRow())).append(';');
2031             }
2032             LOG.trace("Sending requests to [" + e.getKey().getHostnamePort()
2033               + "] with delay of [" + sleepTime + "] for rows [" + sb.toString() + "]");
2034           }
2035           Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>> p =
2036             new Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>>(
2037               e.getValue(), e.getKey(), this.pool.submit(callable));
2038           this.inProgress.addLast(p);
2039         }
2040       }
2041 
2042      /**
2043       * Resubmit the actions which have failed, after a sleep time.
2044       * @throws IOException
2045       */
2046       private void doRetry() throws IOException{
2047         // curNumRetries at this point is 1 or more; decrement to start from 0.
2048         final long sleepTime = ConnectionUtils.getPauseTime(hci.pause, this.curNumRetries - 1);
2049         submit(this.toReplay, sleepTime);
2050         this.toReplay.clear();
2051       }
2052 
2053       /**
2054        * Parameterized batch processing, allowing varying return types for
2055        * different {@link Row} implementations.
2056        * Throws an exception on error. If there are no exceptions, it means that the 'results'
2057        *  array is clean.
2058        */
2059       private void processBatchCallback() throws IOException, InterruptedException {
2060         if (this.results.length != this.rows.size()) {
2061           throw new IllegalArgumentException(
2062             "argument results (size="+results.length+") must be the same size as " +
2063               "argument list (size="+this.rows.size()+")");
2064         }
2065         if (this.rows.isEmpty()) {
2066           return;
2067         }
2068 
2069         boolean isTraceEnabled = LOG.isTraceEnabled();
2070         BatchErrors errors = new BatchErrors();
2071         BatchErrors retriedErrors = null;
2072         if (isTraceEnabled) {
2073           retriedErrors = new BatchErrors();
2074         }
2075 
2076         // We keep the number of retry per action.
2077         int[] nbRetries = new int[this.results.length];
2078 
2079         // Build the action list. This list won't change after being created, hence the
2080         //  indexes will remain constant, allowing a direct lookup.
2081         final List<Action<R>> listActions = new ArrayList<Action<R>>(this.rows.size());
2082         for (int i = 0; i < this.rows.size(); i++) {
2083           Action<R> action = new Action<R>(this.rows.get(i), i);
2084           listActions.add(action);
2085         }
2086 
2087         // execute the actions. We will analyze and resubmit the actions in a 'while' loop.
2088         submit(listActions, 0);
2089 
2090         // LastRetry is true if, either:
2091         //  we had an exception 'DoNotRetry'
2092         //  we had more than numRetries for any action
2093         //  In this case, we will finish the current retries but we won't start new ones.
2094         boolean lastRetry = false;
2095         // despite its name numRetries means number of tries. So if numRetries == 1 it means we
2096         //  won't retry. And we compare vs. 2 in case someone set it to zero.
2097         boolean noRetry = (hci.numRetries < 2);
2098 
2099         // Analyze and resubmit until all actions are done successfully or failed after numRetries
2100         while (!this.inProgress.isEmpty()) {
2101           // We need the original multi action to find out what actions to replay if
2102           //  we have a 'total' failure of the Future<MultiResponse>
2103           // We need the HRegionLocation as we give it back if we go out of retries
2104           Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>> currentTask =
2105             removeFirstDone();
2106 
2107           // Get the answer, keep the exception if any as we will use it for the analysis
2108           MultiResponse responses = null;
2109           ExecutionException exception = null;
2110           try {
2111             responses = currentTask.getThird().get();
2112           } catch (ExecutionException e) {
2113             exception = e;
2114           }
2115 
2116           // Error case: no result at all for this multi action. We need to redo all actions
2117           if (responses == null) {
2118             for (List<Action<R>> actions : currentTask.getFirst().actions.values()) {
2119               for (Action<R> action : actions) {
2120                 Row row = action.getAction();
2121                 // Do not use the exception for updating cache because it might be coming from
2122                 // any of the regions in the MultiAction.
2123                 hci.updateCachedLocations(tableName, row, null, currentTask.getSecond());
2124                 if (noRetry) {
2125                   errors.add(exception, row, currentTask);
2126                 } else {
2127                   if (isTraceEnabled) {
2128                     retriedErrors.add(exception, row, currentTask);
2129                   }
2130                   lastRetry = addToReplay(nbRetries, action);
2131                 }
2132               }
2133             }
2134           } else { // Success or partial success
2135             // Analyze detailed results. We can still have individual failures to be redo.
2136             // two specific exceptions are managed:
2137             //  - DoNotRetryIOException: we continue to retry for other actions
2138             //  - RegionMovedException: we update the cache with the new region location
2139             for (Entry<byte[], List<Pair<Integer, Object>>> resultsForRS :
2140                 responses.getResults().entrySet()) {
2141               for (Pair<Integer, Object> regionResult : resultsForRS.getValue()) {
2142                 Action<R> correspondingAction = listActions.get(regionResult.getFirst());
2143                 Object result = regionResult.getSecond();
2144                 this.results[correspondingAction.getOriginalIndex()] = result;
2145 
2146                 // Failure: retry if it's make sense else update the errors lists
2147                 if (result == null || result instanceof Throwable) {
2148                   Row row = correspondingAction.getAction();
2149                   hci.updateCachedLocations(this.tableName, row, result, currentTask.getSecond());
2150                   if (result instanceof DoNotRetryIOException || noRetry) {
2151                     errors.add((Exception)result, row, currentTask);
2152                   } else {
2153                     if (isTraceEnabled) {
2154                       retriedErrors.add((Exception)result, row, currentTask);
2155                     }
2156                     lastRetry = addToReplay(nbRetries, correspondingAction);
2157                   }
2158                 } else // success
2159                   if (callback != null) {
2160                     this.callback.update(resultsForRS.getKey(),
2161                       this.rows.get(regionResult.getFirst()).getRow(), (R) result);
2162                 }
2163               }
2164             }
2165           }
2166 
2167           // Retry all actions in toReplay then clear it.
2168           if (!noRetry && !toReplay.isEmpty()) {
2169             if (isTraceEnabled) {
2170               LOG.trace("Retrying due to errors" + (lastRetry ? " (one last time)" : "")
2171                    + ": " + retriedErrors.getDescriptionAndClear());
2172             }
2173             doRetry();
2174             if (lastRetry) {
2175               noRetry = true;
2176             }
2177           }
2178         }
2179 
2180         errors.rethrowIfAny();
2181       }
2182 
2183 
2184       private class BatchErrors {
2185         private List<Throwable> exceptions = new ArrayList<Throwable>();
2186         private List<Row> actions = new ArrayList<Row>();
2187         private List<String> addresses = new ArrayList<String>();
2188 
2189         public void add(Exception ex, Row row,
2190           Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>> obj) {
2191           exceptions.add(ex);
2192           actions.add(row);
2193           addresses.add(obj.getSecond().getHostnamePort());
2194         }
2195 
2196         public void rethrowIfAny() throws RetriesExhaustedWithDetailsException {
2197           if (!exceptions.isEmpty()) {
2198             throw makeException();
2199           }
2200         }
2201 
2202         public String getDescriptionAndClear(){
2203           if (exceptions.isEmpty()) {
2204             return "";
2205           }
2206           String result = makeException().getExhaustiveDescription();
2207           exceptions.clear();
2208           actions.clear();
2209           addresses.clear();
2210           return result;
2211         }
2212 
2213         private RetriesExhaustedWithDetailsException makeException() {
2214           return new RetriesExhaustedWithDetailsException(exceptions, actions, addresses);
2215         }
2216       }
2217 
2218       /**
2219        * Put the action that has to be retried in the Replay list.
2220        * @return true if we're out of numRetries and it's the last retry.
2221        */
2222       private boolean addToReplay(int[] nbRetries, Action<R> action) {
2223         this.toReplay.add(action);
2224         nbRetries[action.getOriginalIndex()]++;
2225         if (nbRetries[action.getOriginalIndex()] > this.curNumRetries) {
2226           this.curNumRetries = nbRetries[action.getOriginalIndex()];
2227         }
2228         // numRetries means number of tries, while curNumRetries means current number of retries. So
2229         //  we need to add 1 to make them comparable. And as we look for the last try we compare
2230         //  with '>=' and no '>'. And we need curNumRetries to means what it says as we don't want
2231         //  to initialize it to 1.
2232         return ( (this.curNumRetries +1) >= hci.numRetries);
2233       }
2234 
2235       /**
2236        * Wait for one of tasks to be done, and remove it from the list.
2237        * @return the tasks done.
2238        */
2239       private Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>>
2240       removeFirstDone() throws InterruptedException {
2241         while (true) {
2242           synchronized (finishedTasks) {
2243             if (!finishedTasks.isEmpty()) {
2244               MultiAction<R> done = finishedTasks.remove(finishedTasks.size() - 1);
2245 
2246               // We now need to remove it from the inProgress part.
2247               Iterator<Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>>> it =
2248                 inProgress.iterator();
2249               while (it.hasNext()) {
2250                 Triple<MultiAction<R>, HRegionLocation, Future<MultiResponse>> task = it.next();
2251                 if (task.getFirst() == done) { // We have the exact object. No java equals here.
2252                   it.remove();
2253                   return task;
2254                 }
2255               }
2256               LOG.error("Development error: We didn't see a task in the list. " +
2257                 done.getRegions());
2258             }
2259             finishedTasks.wait(10);
2260           }
2261         }
2262       }
2263 
2264       private Callable<MultiResponse> createDelayedCallable(
2265         final long delay, final HRegionLocation loc, final MultiAction<R> multi) {
2266 
2267         final Callable<MultiResponse> delegate = hci.createCallable(loc, multi, tableName);
2268 
2269         return new Callable<MultiResponse>() {
2270           private final long creationTime = System.currentTimeMillis();
2271 
2272           @Override
2273           public MultiResponse call() throws Exception {
2274             try {
2275               final long waitingTime = delay + creationTime - System.currentTimeMillis();
2276               if (waitingTime > 0) {
2277                 Thread.sleep(waitingTime);
2278               }
2279               return delegate.call();
2280             } finally {
2281               synchronized (finishedTasks) {
2282                 finishedTasks.add(multi);
2283                 finishedTasks.notifyAll();
2284               }
2285             }
2286           }
2287         };
2288       }
2289     }
2290 
2291     /*
2292      * Return the number of cached region for a table. It will only be called
2293      * from a unit test.
2294      */
2295     int getNumberOfCachedRegionLocations(final byte[] tableName) {
2296       Integer key = Bytes.mapKey(tableName);
2297       synchronized (this.cachedRegionLocations) {
2298         Map<byte[], HRegionLocation> tableLocs = this.cachedRegionLocations.get(key);
2299         if (tableLocs == null) {
2300           return 0;
2301         }
2302         return tableLocs.values().size();
2303       }
2304     }
2305 
2306     /**
2307      * Check the region cache to see whether a region is cached yet or not.
2308      * Called by unit tests.
2309      * @param tableName tableName
2310      * @param row row
2311      * @return Region cached or not.
2312      */
2313     boolean isRegionCached(final byte[] tableName, final byte[] row) {
2314       HRegionLocation location = getCachedLocation(tableName, row);
2315       return location != null;
2316     }
2317 
2318     @Override
2319     public void setRegionCachePrefetch(final byte[] tableName,
2320         final boolean enable) {
2321       if (!enable) {
2322         regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName));
2323       }
2324       else {
2325         regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName));
2326       }
2327     }
2328 
2329     @Override
2330     public boolean getRegionCachePrefetch(final byte[] tableName) {
2331       return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName));
2332     }
2333 
2334     @Override
2335     public void abort(final String msg, Throwable t) {
2336       if (t instanceof KeeperException.SessionExpiredException
2337         && keepAliveZookeeper != null) {
2338         synchronized (masterAndZKLock) {
2339           if (keepAliveZookeeper != null) {
2340             LOG.warn("This client just lost it's session with ZooKeeper," +
2341               " closing it." +
2342               " It will be recreated next time someone needs it", t);
2343             closeZooKeeperWatcher();
2344           }
2345         }
2346       } else {
2347         if (t != null) {
2348           LOG.fatal(msg, t);
2349         } else {
2350           LOG.fatal(msg);
2351         }
2352         this.aborted = true;
2353         close();
2354         this.closed = true;
2355       }
2356     }
2357 
2358     @Override
2359     public boolean isClosed() {
2360       return this.closed;
2361     }
2362 
2363     @Override
2364     public boolean isAborted(){
2365       return this.aborted;
2366     }
2367 
2368     @Override
2369     public int getCurrentNrHRS() throws IOException {
2370       ZooKeeperKeepAliveConnection zkw = getKeepAliveZooKeeperWatcher();
2371 
2372       try {
2373         // We go to zk rather than to master to get count of regions to avoid
2374         // HTable having a Master dependency.  See HBase-2828
2375         return ZKUtil.getNumberOfChildren(zkw, zkw.rsZNode);
2376       } catch (KeeperException ke) {
2377         throw new IOException("Unexpected ZooKeeper exception", ke);
2378       } finally {
2379           zkw.close();
2380       }
2381     }
2382 
2383     /**
2384      * Increment this client's reference count.
2385      */
2386     void incCount() {
2387       ++refCount;
2388     }
2389 
2390     /**
2391      * Decrement this client's reference count.
2392      */
2393     void decCount() {
2394       if (refCount > 0) {
2395         --refCount;
2396       }
2397     }
2398 
2399     /**
2400      * Return if this client has no reference
2401      *
2402      * @return true if this client has no reference; false otherwise
2403      */
2404     boolean isZeroReference() {
2405       return refCount == 0;
2406     }
2407 
2408     void internalClose() {
2409       if (this.closed) {
2410         return;
2411       }
2412       delayedClosing.stop("Closing connection");
2413       closeMaster();
2414       closeZooKeeperWatcher();
2415       this.servers.clear();
2416       this.rpcEngine.close();
2417       if (clusterStatusListener != null) {
2418         clusterStatusListener.close();
2419       }
2420       this.closed = true;
2421     }
2422 
2423     @Override
2424     public void close() {
2425       if (managed) {
2426         if (aborted) {
2427           HConnectionManager.deleteStaleConnection(this);
2428         } else {
2429           HConnectionManager.deleteConnection(this, false);
2430         }
2431       } else {
2432         internalClose();
2433       }
2434     }
2435 
2436     /**
2437      * Close the connection for good, regardless of what the current value of
2438      * {@link #refCount} is. Ideally, {@link #refCount} should be zero at this
2439      * point, which would be the case if all of its consumers close the
2440      * connection. However, on the off chance that someone is unable to close
2441      * the connection, perhaps because it bailed out prematurely, the method
2442      * below will ensure that this {@link HConnection} instance is cleaned up.
2443      * Caveat: The JVM may take an unknown amount of time to call finalize on an
2444      * unreachable object, so our hope is that every consumer cleans up after
2445      * itself, like any good citizen.
2446      */
2447     @Override
2448     protected void finalize() throws Throwable {
2449       super.finalize();
2450       // Pretend as if we are about to release the last remaining reference
2451       refCount = 1;
2452       close();
2453     }
2454 
2455     @Override
2456     public HTableDescriptor[] listTables() throws IOException {
2457       MasterMonitorKeepAliveConnection master = getKeepAliveMasterMonitor();
2458       try {
2459         GetTableDescriptorsRequest req =
2460           RequestConverter.buildGetTableDescriptorsRequest(null);
2461         return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2462       } catch (ServiceException se) {
2463         throw ProtobufUtil.getRemoteException(se);
2464       } finally {
2465         master.close();
2466       }
2467     }
2468 
2469     @Override
2470     public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) throws IOException {
2471       if (tableNames == null || tableNames.isEmpty()) return new HTableDescriptor[0];
2472       MasterMonitorKeepAliveConnection master = getKeepAliveMasterMonitor();
2473       try {
2474         GetTableDescriptorsRequest req =
2475           RequestConverter.buildGetTableDescriptorsRequest(tableNames);
2476         return ProtobufUtil.getHTableDescriptorArray(master.getTableDescriptors(null, req));
2477       } catch (ServiceException se) {
2478         throw ProtobufUtil.getRemoteException(se);
2479       } finally {
2480         master.close();
2481       }
2482     }
2483 
2484     /**
2485      * Connects to the master to get the table descriptor.
2486      * @param tableName table name
2487      * @return
2488      * @throws IOException if the connection to master fails or if the table
2489      *  is not found.
2490      */
2491     @Override
2492     public HTableDescriptor getHTableDescriptor(final byte[] tableName)
2493     throws IOException {
2494       if (tableName == null || tableName.length == 0) return null;
2495       if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
2496         return HTableDescriptor.META_TABLEDESC;
2497       }
2498       MasterMonitorKeepAliveConnection master = getKeepAliveMasterMonitor();
2499       GetTableDescriptorsResponse htds;
2500       try {
2501         GetTableDescriptorsRequest req =
2502           RequestConverter.buildGetTableDescriptorsRequest(null);
2503         htds = master.getTableDescriptors(null, req);
2504       } catch (ServiceException se) {
2505         throw ProtobufUtil.getRemoteException(se);
2506       } finally {
2507         master.close();
2508       }
2509       for (TableSchema ts : htds.getTableSchemaList()) {
2510         if (Bytes.equals(tableName, ts.getName().toByteArray())) {
2511           return HTableDescriptor.convert(ts);
2512         }
2513       }
2514       throw new TableNotFoundException(Bytes.toString(tableName));
2515     }
2516 
2517     /**
2518      * Override the RpcClientEngine implementation used by this connection.
2519      * <strong>FOR TESTING PURPOSES ONLY!</strong>
2520      */
2521     void setRpcEngine(RpcClientEngine engine) {
2522       this.rpcEngine = engine;
2523     }
2524   }
2525 
2526   /**
2527    * Set the number of retries to use serverside when trying to communicate
2528    * with another server over {@link HConnection}.  Used updating catalog
2529    * tables, etc.  Call this method before we create any Connections.
2530    * @param c The Configuration instance to set the retries into.
2531    * @param log Used to log what we set in here.
2532    */
2533   public static void setServerSideHConnectionRetries(final Configuration c,
2534       final Log log) {
2535     int hcRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
2536       HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
2537     // Go big.  Multiply by 10.  If we can't get to meta after this many retries
2538     // then something seriously wrong.
2539     int serversideMultiplier =
2540       c.getInt("hbase.client.serverside.retries.multiplier", 10);
2541     int retries = hcRetries * serversideMultiplier;
2542     c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, retries);
2543     log.debug("HConnection retries=" + retries);
2544   }
2545 }
2546