View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.IOException;
23  import java.lang.reflect.UndeclaredThrowableException;
24  import java.util.ArrayList;
25  import java.util.HashMap;
26  import java.util.LinkedHashMap;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.Map.Entry;
30  import java.util.Set;
31  import java.util.TreeSet;
32  import java.util.concurrent.Callable;
33  import java.util.concurrent.ConcurrentHashMap;
34  import java.util.concurrent.CopyOnWriteArraySet;
35  import java.util.concurrent.ExecutionException;
36  import java.util.concurrent.ExecutorService;
37  import java.util.concurrent.Future;
38  import java.util.concurrent.atomic.AtomicBoolean;
39  import java.util.concurrent.atomic.AtomicInteger;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.hbase.DoNotRetryIOException;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HRegionLocation;
48  import org.apache.hadoop.hbase.HServerAddress;
49  import org.apache.hadoop.hbase.HTableDescriptor;
50  import org.apache.hadoop.hbase.KeyValue;
51  import org.apache.hadoop.hbase.MasterAddressTracker;
52  import org.apache.hadoop.hbase.MasterNotRunningException;
53  import org.apache.hadoop.hbase.RemoteExceptionHandler;
54  import org.apache.hadoop.hbase.TableNotFoundException;
55  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
56  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
57  import org.apache.hadoop.hbase.ipc.HBaseRPC;
58  import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
59  import org.apache.hadoop.hbase.ipc.HMasterInterface;
60  import org.apache.hadoop.hbase.ipc.HRegionInterface;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.Pair;
63  import org.apache.hadoop.hbase.util.SoftValueSortedMap;
64  import org.apache.hadoop.hbase.util.Writables;
65  import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
66  import org.apache.hadoop.hbase.zookeeper.ZKTable;
67  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
68  import org.apache.hadoop.ipc.RemoteException;
69  import org.apache.zookeeper.KeeperException;
70  
71  /**
72   * A non-instantiable class that manages {@link HConnection}s.
73   * This class has a static Map of {@link HConnection} instances keyed by
74   * {@link Configuration}; all invocations of {@link #getConnection(Configuration)}
75   * that pass the same {@link Configuration} instance will be returned the same
76   * {@link  HConnection} instance (Adding properties to a Configuration
77   * instance does not change its object identity).  Sharing {@link HConnection}
78   * instances is usually what you want; all clients of the {@link HConnection}
79   * instances share the HConnections' cache of Region locations rather than each
80   * having to discover for itself the location of meta, root, etc.  It makes
81   * sense for the likes of the pool of HTables class {@link HTablePool}, for
82   * instance (If concerned that a single {@link HConnection} is insufficient
83   * for sharing amongst clients in say an heavily-multithreaded environment,
84   * in practise its not proven to be an issue.  Besides, {@link HConnection} is
85   * implemented atop Hadoop RPC and as of this writing, Hadoop RPC does a
86   * connection per cluster-member, exclusively).
87   *
88   * <p>But sharing connections
89   * makes clean up of {@link HConnection} instances a little awkward.  Currently,
90   * clients cleanup by calling
91   * {@link #deleteConnection(Configuration, boolean)}.  This will shutdown the
92   * zookeeper connection the HConnection was using and clean up all
93   * HConnection resources as well as stopping proxies to servers out on the
94   * cluster. Not running the cleanup will not end the world; it'll
95   * just stall the closeup some and spew some zookeeper connection failed
96   * messages into the log.  Running the cleanup on a {@link HConnection} that is
97   * subsequently used by another will cause breakage so be careful running
98   * cleanup.
99   * <p>To create a {@link HConnection} that is not shared by others, you can
100  * create a new {@link Configuration} instance, pass this new instance to
101  * {@link #getConnection(Configuration)}, and then when done, close it up by
102  * doing something like the following:
103  * <pre>
104  * {@code
105  * Configuration newConfig = new Configuration(originalConf);
106  * HConnection connection = HConnectionManager.getConnection(newConfig);
107  * // Use the connection to your hearts' delight and then when done...
108  * HConnectionManager.deleteConnection(newConfig, true);
109  * }
110  * </pre>
111  * <p>Cleanup used to be done inside in a shutdown hook.  On startup we'd
112  * register a shutdown hook that called {@link #deleteAllConnections(boolean)}
113  * on its way out but the order in which shutdown hooks run is not defined so
114  * were problematic for clients of HConnection that wanted to register their
115  * own shutdown hooks so we removed ours though this shifts the onus for
116  * cleanup to the client.
117  */
118 @SuppressWarnings("serial")
119 public class HConnectionManager {
120   static final int MAX_CACHED_HBASE_INSTANCES = 31;
121 
122   // A LRU Map of Configuration hashcode -> TableServers. We set instances to 31.
123   // The zk default max connections to the ensemble from the one client is 30 so
124   // should run into zk issues before hit this value of 31.
125   private static final Map<Configuration, HConnectionImplementation> HBASE_INSTANCES =
126     new LinkedHashMap<Configuration, HConnectionImplementation>
127       ((int) (MAX_CACHED_HBASE_INSTANCES/0.75F)+1, 0.75F, true) {
128       @Override
129       protected boolean removeEldestEntry(Map.Entry<Configuration, HConnectionImplementation> eldest) {
130         return size() > MAX_CACHED_HBASE_INSTANCES;
131       }
132   };
133 
134   /*
135    * Non-instantiable.
136    */
137   protected HConnectionManager() {
138     super();
139   }
140 
141   /**
142    * Get the connection that goes with the passed <code>conf</code>
143    * configuration instance.
144    * If no current connection exists, method creates a new connection for the
145    * passed <code>conf</code> instance.
146    * @param conf configuration
147    * @return HConnection object for <code>conf</code>
148    * @throws ZooKeeperConnectionException
149    */
150   public static HConnection getConnection(Configuration conf)
151   throws ZooKeeperConnectionException {
152     HConnectionImplementation connection;
153     synchronized (HBASE_INSTANCES) {
154       connection = HBASE_INSTANCES.get(conf);
155       if (connection == null) {
156         connection = new HConnectionImplementation(conf);
157         HBASE_INSTANCES.put(conf, connection);
158       }
159     }
160     return connection;
161   }
162 
163   /**
164    * Delete connection information for the instance specified by configuration.
165    * This will close connection to the zookeeper ensemble and let go of all
166    * resources.
167    * @param conf configuration whose identity is used to find {@link HConnection}
168    * instance.
169    * @param stopProxy Shuts down all the proxy's put up to cluster members
170    * including to cluster HMaster.  Calls {@link HBaseRPC#stopProxy(org.apache.hadoop.ipc.VersionedProtocol)}.
171    */
172   public static void deleteConnection(Configuration conf, boolean stopProxy) {
173     synchronized (HBASE_INSTANCES) {
174       HConnectionImplementation t = HBASE_INSTANCES.remove(conf);
175       if (t != null) {
176         t.close(stopProxy);
177       }
178     }
179   }
180 
181   /**
182    * Delete information for all connections.
183    * @param stopProxy stop the proxy as well
184    * @throws IOException
185    */
186   public static void deleteAllConnections(boolean stopProxy) {
187     synchronized (HBASE_INSTANCES) {
188       for (HConnectionImplementation t : HBASE_INSTANCES.values()) {
189         if (t != null) {
190           t.close(stopProxy);
191         }
192       }
193     }
194   }
195 
196   /**
197    * It is provided for unit test cases which verify the behavior of region
198    * location cache prefetch.
199    * @return Number of cached regions for the table.
200    * @throws ZooKeeperConnectionException
201    */
202   static int getCachedRegionCount(Configuration conf,
203       byte[] tableName)
204   throws ZooKeeperConnectionException {
205     HConnectionImplementation connection = (HConnectionImplementation)getConnection(conf);
206     return connection.getNumberOfCachedRegionLocations(tableName);
207   }
208 
209   /**
210    * It's provided for unit test cases which verify the behavior of region
211    * location cache prefetch.
212    * @return true if the region where the table and row reside is cached.
213    * @throws ZooKeeperConnectionException
214    */
215   static boolean isRegionCached(Configuration conf,
216       byte[] tableName, byte[] row) throws ZooKeeperConnectionException {
217     HConnectionImplementation connection = (HConnectionImplementation)getConnection(conf);
218     return connection.isRegionCached(tableName, row);
219   }
220 
221   /* Encapsulates connection to zookeeper and regionservers.*/
222   static class HConnectionImplementation implements HConnection {
223     static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);
224     private final Class<? extends HRegionInterface> serverInterfaceClass;
225     private final long pause;
226     private final int numRetries;
227     private final int maxRPCAttempts;
228     private final int rpcTimeout;
229     private final int prefetchRegionLimit;
230 
231     private final Object masterLock = new Object();
232     private volatile boolean closed;
233     private volatile HMasterInterface master;
234     private volatile boolean masterChecked;
235     // ZooKeeper reference
236     private ZooKeeperWatcher zooKeeper;
237     // ZooKeeper-based master address tracker
238     private MasterAddressTracker masterAddressTracker;
239     private RootRegionTracker rootRegionTracker;
240     
241     private final Object metaRegionLock = new Object();
242 
243     private final Object userRegionLock = new Object();
244 
245     private final Configuration conf;
246     // Known region HServerAddress.toString() -> HRegionInterface
247 
248     private final Map<String, HRegionInterface> servers =
249       new ConcurrentHashMap<String, HRegionInterface>();
250     private final ConcurrentHashMap<String, String> connectionLock = new ConcurrentHashMap<String, String>();
251 
252     /**
253      * Map of table to table {@link HRegionLocation}s.  The table key is made
254      * by doing a {@link Bytes#mapKey(byte[])} of the table's name.
255      */
256     private final Map<Integer, SoftValueSortedMap<byte [], HRegionLocation>>
257       cachedRegionLocations =
258         new HashMap<Integer, SoftValueSortedMap<byte [], HRegionLocation>>();
259 
260     // region cache prefetch is enabled by default. this set contains all
261     // tables whose region cache prefetch are disabled.
262     private final Set<Integer> regionCachePrefetchDisabledTables =
263       new CopyOnWriteArraySet<Integer>();
264 
265     /**
266      * constructor
267      * @param conf Configuration object
268      */
269     @SuppressWarnings("unchecked")
270     public HConnectionImplementation(Configuration conf)
271     throws ZooKeeperConnectionException {
272       this.conf = conf;
273       String serverClassName = conf.get(HConstants.REGION_SERVER_CLASS,
274         HConstants.DEFAULT_REGION_SERVER_CLASS);
275       this.closed = false;
276       try {
277         this.serverInterfaceClass =
278           (Class<? extends HRegionInterface>) Class.forName(serverClassName);
279       } catch (ClassNotFoundException e) {
280         throw new UnsupportedOperationException(
281             "Unable to find region server interface " + serverClassName, e);
282       }
283 
284       this.pause = conf.getLong("hbase.client.pause", 1000);
285       this.numRetries = conf.getInt("hbase.client.retries.number", 10);
286       this.maxRPCAttempts = conf.getInt("hbase.client.rpc.maxattempts", 1);
287       this.rpcTimeout = conf.getInt(
288           HConstants.HBASE_RPC_TIMEOUT_KEY,
289           HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
290 
291       this.prefetchRegionLimit = conf.getInt("hbase.client.prefetch.limit",
292           10);
293 
294       setupZookeeperTrackers();
295 
296       this.master = null;
297       this.masterChecked = false;
298     }
299 
300     private synchronized void setupZookeeperTrackers()
301         throws ZooKeeperConnectionException{
302       // initialize zookeeper and master address manager
303       this.zooKeeper = getZooKeeperWatcher();
304       masterAddressTracker = new MasterAddressTracker(this.zooKeeper, this);
305       masterAddressTracker.start();
306 
307       this.rootRegionTracker = new RootRegionTracker(this.zooKeeper, this);
308       this.rootRegionTracker.start();
309     }
310 
311     private synchronized void resetZooKeeperTrackers()
312         throws ZooKeeperConnectionException {
313       LOG.info("Trying to reconnect to zookeeper");
314       masterAddressTracker.stop();
315       masterAddressTracker = null;
316       rootRegionTracker.stop();
317       rootRegionTracker = null;
318       this.zooKeeper = null;
319       setupZookeeperTrackers();
320     }
321 
322     public Configuration getConfiguration() {
323       return this.conf;
324     }
325 
326     private long getPauseTime(int tries) {
327       int ntries = tries;
328       if (ntries >= HConstants.RETRY_BACKOFF.length) {
329         ntries = HConstants.RETRY_BACKOFF.length - 1;
330       }
331       return this.pause * HConstants.RETRY_BACKOFF[ntries];
332     }
333 
334     public HMasterInterface getMaster()
335     throws MasterNotRunningException, ZooKeeperConnectionException {
336 
337       // Check if we already have a good master connection
338       if (master != null) {
339         if (master.isMasterRunning()) {
340           return master;
341         }
342       }
343 
344       HServerAddress masterLocation = null;
345       synchronized (this.masterLock) {
346         for (int tries = 0;
347           !this.closed &&
348           !this.masterChecked && this.master == null &&
349           tries < numRetries;
350         tries++) {
351 
352           try {
353             masterLocation = masterAddressTracker.getMasterAddress();
354             if(masterLocation == null) {
355               LOG.info("ZooKeeper available but no active master location found");
356               throw new MasterNotRunningException();
357             }
358 
359             HMasterInterface tryMaster = (HMasterInterface)HBaseRPC.getProxy(
360                 HMasterInterface.class, HBaseRPCProtocolVersion.versionID,
361                 masterLocation.getInetSocketAddress(), this.conf, this.rpcTimeout);
362 
363             if (tryMaster.isMasterRunning()) {
364               this.master = tryMaster;
365               this.masterLock.notifyAll();
366               break;
367             }
368 
369           } catch (IOException e) {
370             if (tries == numRetries - 1) {
371               // This was our last chance - don't bother sleeping
372               LOG.info("getMaster attempt " + tries + " of " + this.numRetries +
373                 " failed; no more retrying.", e);
374               break;
375             }
376             LOG.info("getMaster attempt " + tries + " of " + this.numRetries +
377               " failed; retrying after sleep of " +
378               getPauseTime(tries), e);
379           }
380 
381           // Cannot connect to master or it is not running. Sleep & retry
382           try {
383             this.masterLock.wait(getPauseTime(tries));
384           } catch (InterruptedException e) {
385             Thread.currentThread().interrupt();
386             throw new RuntimeException("Thread was interrupted while trying to connect to master.");
387           }
388         }
389         this.masterChecked = true;
390       }
391       if (this.master == null) {
392         if (masterLocation == null) {
393           throw new MasterNotRunningException();
394         }
395         throw new MasterNotRunningException(masterLocation.toString());
396       }
397       return this.master;
398     }
399 
400     public boolean isMasterRunning()
401     throws MasterNotRunningException, ZooKeeperConnectionException {
402       if (this.master == null) {
403         getMaster();
404       }
405       boolean isRunning = master.isMasterRunning();
406       if(isRunning) {
407         return true;
408       }
409       throw new MasterNotRunningException();
410     }
411 
412     public HRegionLocation getRegionLocation(final byte [] name,
413         final byte [] row, boolean reload)
414     throws IOException {
415       return reload? relocateRegion(name, row): locateRegion(name, row);
416     }
417 
418     public HTableDescriptor[] listTables() throws IOException {
419       final TreeSet<HTableDescriptor> uniqueTables =
420         new TreeSet<HTableDescriptor>();
421       MetaScannerVisitor visitor = new MetaScannerVisitor() {
422         public boolean processRow(Result result) throws IOException {
423           try {
424             byte[] value = result.getValue(HConstants.CATALOG_FAMILY,
425                 HConstants.REGIONINFO_QUALIFIER);
426             HRegionInfo info = null;
427             if (value != null) {
428               info = Writables.getHRegionInfo(value);
429             }
430             // Only examine the rows where the startKey is zero length
431             if (info != null && info.getStartKey().length == 0) {
432               uniqueTables.add(info.getTableDesc());
433             }
434             return true;
435           } catch (RuntimeException e) {
436             LOG.error("Result=" + result);
437             throw e;
438           }
439         }
440       };
441       MetaScanner.metaScan(conf, visitor);
442       return uniqueTables.toArray(new HTableDescriptor[uniqueTables.size()]);
443     }
444 
445     public boolean isTableEnabled(byte[] tableName) throws IOException {
446       return testTableOnlineState(tableName, true);
447     }
448 
449     public boolean isTableDisabled(byte[] tableName) throws IOException {
450       return testTableOnlineState(tableName, false);
451     }
452 
453     public boolean isTableAvailable(final byte[] tableName) throws IOException {
454       final AtomicBoolean available = new AtomicBoolean(true);
455       final AtomicInteger regionCount = new AtomicInteger(0);
456       MetaScannerVisitor visitor = new MetaScannerVisitor() {
457         @Override
458         public boolean processRow(Result row) throws IOException {
459           byte[] value = row.getValue(HConstants.CATALOG_FAMILY,
460               HConstants.REGIONINFO_QUALIFIER);
461           HRegionInfo info = Writables.getHRegionInfoOrNull(value);
462           if (info != null) {
463             if (Bytes.equals(tableName, info.getTableDesc().getName())) {
464               value = row.getValue(HConstants.CATALOG_FAMILY,
465                   HConstants.SERVER_QUALIFIER);
466               if (value == null) {
467                 available.set(false);
468                 return false;
469               }
470               regionCount.incrementAndGet();
471             }
472           }
473           return true;
474         }
475       };
476       MetaScanner.metaScan(conf, visitor);
477       return available.get() && (regionCount.get() > 0);
478     }
479 
480     /*
481      * @param True if table is online
482      */
483     private boolean testTableOnlineState(byte [] tableName, boolean online)
484     throws IOException {
485       if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
486         // The root region is always enabled
487         return online;
488       }
489       String tableNameStr = Bytes.toString(tableName);
490       try {
491         if (online) {
492           return ZKTable.isEnabledTable(this.zooKeeper, tableNameStr);
493         }
494         return ZKTable.isDisabledTable(this.zooKeeper, tableNameStr);
495       } catch (KeeperException e) {
496         throw new IOException("Enable/Disable failed", e);
497       }
498     }
499 
500     private static class HTableDescriptorFinder
501     implements MetaScanner.MetaScannerVisitor {
502         byte[] tableName;
503         HTableDescriptor result;
504         protected HTableDescriptorFinder(byte[] tableName) {
505           this.tableName = tableName;
506         }
507         public boolean processRow(Result rowResult) throws IOException {
508           HRegionInfo info = Writables.getHRegionInfoOrNull(
509               rowResult.getValue(HConstants.CATALOG_FAMILY,
510                   HConstants.REGIONINFO_QUALIFIER));
511           if (info == null) return true;
512           HTableDescriptor desc = info.getTableDesc();
513           if (Bytes.compareTo(desc.getName(), tableName) == 0) {
514             result = desc;
515             return false;
516           }
517           return true;
518         }
519         HTableDescriptor getResult() {
520           return result;
521         }
522     }
523 
524     public HTableDescriptor getHTableDescriptor(final byte[] tableName)
525     throws IOException {
526       if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
527         return new UnmodifyableHTableDescriptor(HTableDescriptor.ROOT_TABLEDESC);
528       }
529       if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
530         return HTableDescriptor.META_TABLEDESC;
531       }
532       HTableDescriptorFinder finder = new HTableDescriptorFinder(tableName);
533       MetaScanner.metaScan(conf, finder, tableName);
534       HTableDescriptor result = finder.getResult();
535       if (result == null) {
536         throw new TableNotFoundException(Bytes.toString(tableName));
537       }
538       return result;
539     }
540 
541     @Override
542     public HRegionLocation locateRegion(final byte [] regionName)
543     throws IOException {
544       // TODO implement.  use old stuff or new stuff?
545       return null;
546     }
547 
548     @Override
549     public List<HRegionLocation> locateRegions(final byte [] tableName)
550     throws IOException {
551       // TODO implement.  use old stuff or new stuff?
552       return null;
553     }
554 
555     public HRegionLocation locateRegion(final byte [] tableName,
556         final byte [] row)
557     throws IOException{
558       return locateRegion(tableName, row, true);
559     }
560 
561     public HRegionLocation relocateRegion(final byte [] tableName,
562         final byte [] row)
563     throws IOException{
564       return locateRegion(tableName, row, false);
565     }
566 
567     private HRegionLocation locateRegion(final byte [] tableName,
568       final byte [] row, boolean useCache)
569     throws IOException {
570       if (this.closed) throw new IOException(toString() + " closed");
571       if (tableName == null || tableName.length == 0) {
572         throw new IllegalArgumentException(
573             "table name cannot be null or zero length");
574       }
575 
576       if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
577         try {
578           HServerAddress hsa =
579             this.rootRegionTracker.waitRootRegionLocation(this.rpcTimeout);
580           LOG.debug("Lookedup root region location, connection=" + this +
581             "; hsa=" + hsa);
582           if (hsa == null) return null;
583           return new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, hsa);
584         } catch (InterruptedException e) {
585           Thread.currentThread().interrupt();
586           return null;
587         }
588       } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
589         return locateRegionInMeta(HConstants.ROOT_TABLE_NAME, tableName, row,
590             useCache, metaRegionLock);
591       } else {
592         // Region not in the cache - have to go to the meta RS
593         return locateRegionInMeta(HConstants.META_TABLE_NAME, tableName, row,
594             useCache, userRegionLock);
595       }
596     }
597 
598     /*
599      * Search .META. for the HRegionLocation info that contains the table and
600      * row we're seeking. It will prefetch certain number of regions info and
601      * save them to the global region cache.
602      */
603     private void prefetchRegionCache(final byte[] tableName,
604         final byte[] row) {
605       // Implement a new visitor for MetaScanner, and use it to walk through
606       // the .META.
607       MetaScannerVisitor visitor = new MetaScannerVisitor() {
608         public boolean processRow(Result result) throws IOException {
609           try {
610             byte[] value = result.getValue(HConstants.CATALOG_FAMILY,
611                 HConstants.REGIONINFO_QUALIFIER);
612             HRegionInfo regionInfo = null;
613 
614             if (value != null) {
615               // convert the row result into the HRegionLocation we need!
616               regionInfo = Writables.getHRegionInfo(value);
617 
618               // possible we got a region of a different table...
619               if (!Bytes.equals(regionInfo.getTableDesc().getName(),
620                   tableName)) {
621                 return false; // stop scanning
622               }
623               if (regionInfo.isOffline()) {
624                 // don't cache offline regions
625                 return true;
626               }
627               value = result.getValue(HConstants.CATALOG_FAMILY,
628                   HConstants.SERVER_QUALIFIER);
629               if (value == null) {
630                 return true;  // don't cache it
631               }
632               final String serverAddress = Bytes.toString(value);
633 
634               // instantiate the location
635               HRegionLocation loc = new HRegionLocation(regionInfo,
636                 new HServerAddress(serverAddress));
637               // cache this meta entry
638               cacheLocation(tableName, loc);
639             }
640             return true;
641           } catch (RuntimeException e) {
642             throw new IOException(e);
643           }
644         }
645       };
646       try {
647         // pre-fetch certain number of regions info at region cache.
648         MetaScanner.metaScan(conf, visitor, tableName, row,
649             this.prefetchRegionLimit);
650       } catch (IOException e) {
651         LOG.warn("Encountered problems when prefetch META table: ", e);
652       }
653     }
654 
655     /*
656       * Search one of the meta tables (-ROOT- or .META.) for the HRegionLocation
657       * info that contains the table and row we're seeking.
658       */
659     private HRegionLocation locateRegionInMeta(final byte [] parentTable,
660       final byte [] tableName, final byte [] row, boolean useCache,
661       Object regionLockObject)
662     throws IOException {
663       HRegionLocation location;
664       // If we are supposed to be using the cache, look in the cache to see if
665       // we already have the region.
666       if (useCache) {
667         location = getCachedLocation(tableName, row);
668         if (location != null) {
669           return location;
670         }
671       }
672 
673       // build the key of the meta region we should be looking for.
674       // the extra 9's on the end are necessary to allow "exact" matches
675       // without knowing the precise region names.
676       byte [] metaKey = HRegionInfo.createRegionName(tableName, row,
677         HConstants.NINES, false);
678       for (int tries = 0; true; tries++) {
679         if (tries >= numRetries) {
680           throw new NoServerForRegionException("Unable to find region for "
681             + Bytes.toStringBinary(row) + " after " + numRetries + " tries.");
682         }
683 
684         HRegionLocation metaLocation = null;
685         try {
686           // locate the root or meta region
687           metaLocation = locateRegion(parentTable, metaKey);
688           // If null still, go around again.
689           if (metaLocation == null) continue;
690           HRegionInterface server =
691             getHRegionConnection(metaLocation.getServerAddress());
692 
693           Result regionInfoRow = null;
694           // This block guards against two threads trying to load the meta
695           // region at the same time. The first will load the meta region and
696           // the second will use the value that the first one found.
697           synchronized (regionLockObject) {
698             // If the parent table is META, we may want to pre-fetch some
699             // region info into the global region cache for this table.
700             if (Bytes.equals(parentTable, HConstants.META_TABLE_NAME) &&
701                 (getRegionCachePrefetch(tableName)) )  {
702               prefetchRegionCache(tableName, row);
703             }
704 
705             // Check the cache again for a hit in case some other thread made the
706             // same query while we were waiting on the lock. If not supposed to
707             // be using the cache, delete any existing cached location so it won't
708             // interfere.
709             if (useCache) {
710               location = getCachedLocation(tableName, row);
711               if (location != null) {
712                 return location;
713               }
714             } else {
715               deleteCachedLocation(tableName, row);
716             }
717 
718           // Query the root or meta region for the location of the meta region
719             regionInfoRow = server.getClosestRowBefore(
720             metaLocation.getRegionInfo().getRegionName(), metaKey,
721             HConstants.CATALOG_FAMILY);
722           }
723           if (regionInfoRow == null) {
724             throw new TableNotFoundException(Bytes.toString(tableName));
725           }
726           byte[] value = regionInfoRow.getValue(HConstants.CATALOG_FAMILY,
727               HConstants.REGIONINFO_QUALIFIER);
728           if (value == null || value.length == 0) {
729             throw new IOException("HRegionInfo was null or empty in " +
730               Bytes.toString(parentTable) + ", row=" + regionInfoRow);
731           }
732           // convert the row result into the HRegionLocation we need!
733           HRegionInfo regionInfo = (HRegionInfo) Writables.getWritable(
734               value, new HRegionInfo());
735           // possible we got a region of a different table...
736           if (!Bytes.equals(regionInfo.getTableDesc().getName(), tableName)) {
737             throw new TableNotFoundException(
738               "Table '" + Bytes.toString(tableName) + "' was not found.");
739           }
740           if (regionInfo.isOffline()) {
741             throw new RegionOfflineException("region offline: " +
742               regionInfo.getRegionNameAsString());
743           }
744 
745           value = regionInfoRow.getValue(HConstants.CATALOG_FAMILY,
746               HConstants.SERVER_QUALIFIER);
747           String serverAddress = "";
748           if(value != null) {
749             serverAddress = Bytes.toString(value);
750           }
751           if (serverAddress.equals("")) {
752             throw new NoServerForRegionException("No server address listed " +
753               "in " + Bytes.toString(parentTable) + " for region " +
754               regionInfo.getRegionNameAsString());
755           }
756 
757           // instantiate the location
758           location = new HRegionLocation(regionInfo,
759             new HServerAddress(serverAddress));
760           cacheLocation(tableName, location);
761           return location;
762         } catch (TableNotFoundException e) {
763           // if we got this error, probably means the table just plain doesn't
764           // exist. rethrow the error immediately. this should always be coming
765           // from the HTable constructor.
766           throw e;
767         } catch (IOException e) {
768           if (e instanceof RemoteException) {
769             e = RemoteExceptionHandler.decodeRemoteException(
770                 (RemoteException) e);
771           }
772           if (tries < numRetries - 1) {
773             if (LOG.isDebugEnabled()) {
774               LOG.debug("locateRegionInMeta parentTable=" +
775                 Bytes.toString(parentTable) + ", metaLocation=" +
776                 ((metaLocation == null)? "null": metaLocation) + ", attempt=" +
777                 tries + " of " +
778                 this.numRetries + " failed; retrying after sleep of " +
779                 getPauseTime(tries) + " because: " + e.getMessage());
780             }
781           } else {
782             throw e;
783           }
784           // Only relocate the parent region if necessary
785           if(!(e instanceof RegionOfflineException ||
786               e instanceof NoServerForRegionException)) {
787             relocateRegion(parentTable, metaKey);
788           }
789         }
790         try{
791           Thread.sleep(getPauseTime(tries));
792         } catch (InterruptedException e) {
793           Thread.currentThread().interrupt();
794           throw new IOException("Giving up trying to location region in " +
795             "meta: thread is interrupted.");
796         }
797       }
798     }
799 
800     /*
801      * Search the cache for a location that fits our table and row key.
802      * Return null if no suitable region is located. TODO: synchronization note
803      *
804      * <p>TODO: This method during writing consumes 15% of CPU doing lookup
805      * into the Soft Reference SortedMap.  Improve.
806      *
807      * @param tableName
808      * @param row
809      * @return Null or region location found in cache.
810      */
811     HRegionLocation getCachedLocation(final byte [] tableName,
812         final byte [] row) {
813       SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
814         getTableLocations(tableName);
815 
816       // start to examine the cache. we can only do cache actions
817       // if there's something in the cache for this table.
818       if (tableLocations.isEmpty()) {
819         return null;
820       }
821 
822       HRegionLocation rl = tableLocations.get(row);
823       if (rl != null) {
824         if (LOG.isDebugEnabled()) {
825           LOG.debug("Cache hit for row <" +
826             Bytes.toStringBinary(row) +
827             "> in tableName " + Bytes.toString(tableName) +
828             ": location server " + rl.getServerAddress() +
829             ", location region name " +
830             rl.getRegionInfo().getRegionNameAsString());
831         }
832         return rl;
833       }
834 
835       // Cut the cache so that we only get the part that could contain
836       // regions that match our key
837       SoftValueSortedMap<byte[], HRegionLocation> matchingRegions =
838         tableLocations.headMap(row);
839 
840       // if that portion of the map is empty, then we're done. otherwise,
841       // we need to examine the cached location to verify that it is
842       // a match by end key as well.
843       if (!matchingRegions.isEmpty()) {
844         HRegionLocation possibleRegion =
845           matchingRegions.get(matchingRegions.lastKey());
846 
847         // there is a possibility that the reference was garbage collected
848         // in the instant since we checked isEmpty().
849         if (possibleRegion != null) {
850           byte[] endKey = possibleRegion.getRegionInfo().getEndKey();
851 
852           // make sure that the end key is greater than the row we're looking
853           // for, otherwise the row actually belongs in the next region, not
854           // this one. the exception case is when the endkey is
855           // HConstants.EMPTY_START_ROW, signifying that the region we're
856           // checking is actually the last region in the table.
857           if (Bytes.equals(endKey, HConstants.EMPTY_END_ROW) ||
858               KeyValue.getRowComparator(tableName).compareRows(endKey, 0, endKey.length,
859                   row, 0, row.length) > 0) {
860             return possibleRegion;
861           }
862         }
863       }
864 
865       // Passed all the way through, so we got nothin - complete cache miss
866       return null;
867     }
868 
869     /**
870      * Delete a cached location
871      * @param tableName tableName
872      * @param row
873      */
874     void deleteCachedLocation(final byte [] tableName, final byte [] row) {
875       synchronized (this.cachedRegionLocations) {
876         SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
877             getTableLocations(tableName);
878         // start to examine the cache. we can only do cache actions
879         // if there's something in the cache for this table.
880         if (!tableLocations.isEmpty()) {
881           HRegionLocation rl = getCachedLocation(tableName, row);
882           if (rl != null) {
883             tableLocations.remove(rl.getRegionInfo().getStartKey());
884             if (LOG.isDebugEnabled()) {
885               LOG.debug("Removed " +
886                 rl.getRegionInfo().getRegionNameAsString() +
887                 " for tableName=" + Bytes.toString(tableName) +
888                 " from cache " + "because of " + Bytes.toStringBinary(row));
889             }
890           }
891         }
892       }
893     }
894 
895     /*
896      * @param tableName
897      * @return Map of cached locations for passed <code>tableName</code>
898      */
899     private SoftValueSortedMap<byte [], HRegionLocation> getTableLocations(
900         final byte [] tableName) {
901       // find the map of cached locations for this table
902       Integer key = Bytes.mapKey(tableName);
903       SoftValueSortedMap<byte [], HRegionLocation> result;
904       synchronized (this.cachedRegionLocations) {
905         result = this.cachedRegionLocations.get(key);
906         // if tableLocations for this table isn't built yet, make one
907         if (result == null) {
908           result = new SoftValueSortedMap<byte [], HRegionLocation>(
909               Bytes.BYTES_COMPARATOR);
910           this.cachedRegionLocations.put(key, result);
911         }
912       }
913       return result;
914     }
915 
916     @Override
917     public void clearRegionCache() {
918       synchronized(this.cachedRegionLocations) {
919         this.cachedRegionLocations.clear();
920       }
921     }
922 
923     @Override
924     public void clearRegionCache(final byte [] tableName) {
925       synchronized (this.cachedRegionLocations) {
926         this.cachedRegionLocations.remove(Bytes.mapKey(tableName));
927       }
928     }
929 
930     /*
931      * Put a newly discovered HRegionLocation into the cache.
932      */
933     private void cacheLocation(final byte [] tableName,
934         final HRegionLocation location) {
935       byte [] startKey = location.getRegionInfo().getStartKey();
936       SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
937         getTableLocations(tableName);
938       if (tableLocations.put(startKey, location) == null) {
939         LOG.debug("Cached location for " +
940             location.getRegionInfo().getRegionNameAsString() +
941             " is " + location.getServerAddress());
942       }
943     }
944 
945     public HRegionInterface getHRegionConnection(
946         HServerAddress regionServer, boolean getMaster)
947     throws IOException {
948       if (getMaster) {
949         getMaster();
950       }
951       HRegionInterface server;
952       String rsName = regionServer.toString();
953       // See if we already have a connection (common case)
954       server = this.servers.get(rsName);
955       if (server == null) {
956         // create a unique lock for this RS (if necessary)
957         this.connectionLock.putIfAbsent(rsName, rsName);
958         // get the RS lock
959         synchronized (this.connectionLock.get(rsName)) {
960           // do one more lookup in case we were stalled above
961           server = this.servers.get(rsName);
962           if (server == null) {
963             try {
964               // definitely a cache miss. establish an RPC for this RS
965               server = (HRegionInterface) HBaseRPC.waitForProxy(
966                   serverInterfaceClass, HBaseRPCProtocolVersion.versionID,
967                   regionServer.getInetSocketAddress(), this.conf,
968                   this.maxRPCAttempts, this.rpcTimeout, this.rpcTimeout);
969               this.servers.put(rsName, server);
970             } catch (RemoteException e) {
971               LOG.warn("RemoteException connecting to RS", e);
972               // Throw what the RemoteException was carrying.
973               throw RemoteExceptionHandler.decodeRemoteException(e);
974             }
975           }
976         }
977       }
978       return server;
979     }
980 
981     public HRegionInterface getHRegionConnection(
982         HServerAddress regionServer)
983     throws IOException {
984       return getHRegionConnection(regionServer, false);
985     }
986 
987     /**
988      * Get the ZooKeeper instance for this TableServers instance.
989      *
990      * If ZK has not been initialized yet, this will connect to ZK.
991      * @returns zookeeper reference
992      * @throws ZooKeeperConnectionException if there's a problem connecting to zk
993      */
994     public synchronized ZooKeeperWatcher getZooKeeperWatcher()
995         throws ZooKeeperConnectionException {
996       if(zooKeeper == null) {
997         try {
998           this.zooKeeper = new ZooKeeperWatcher(conf, "hconnection", this);
999         } catch (IOException e) {
1000           throw new ZooKeeperConnectionException(e);
1001         }
1002       }
1003       return zooKeeper;
1004     }
1005 
1006     public <T> T getRegionServerWithRetries(ServerCallable<T> callable)
1007     throws IOException, RuntimeException {
1008       List<Throwable> exceptions = new ArrayList<Throwable>();
1009       for(int tries = 0; tries < numRetries; tries++) {
1010         try {
1011           callable.instantiateServer(tries != 0);
1012           return callable.call();
1013         } catch (Throwable t) {
1014           t = translateException(t);
1015           exceptions.add(t);
1016           if (tries == numRetries - 1) {
1017             throw new RetriesExhaustedException(callable.getServerName(),
1018                 callable.getRegionName(), callable.getRow(), tries, exceptions);
1019           }
1020         }
1021         try {
1022           Thread.sleep(getPauseTime(tries));
1023         } catch (InterruptedException e) {
1024           Thread.currentThread().interrupt();
1025           throw new IOException("Giving up trying to get region server: thread is interrupted.");
1026         }
1027       }
1028       return null;
1029     }
1030 
1031     public <T> T getRegionServerWithoutRetries(ServerCallable<T> callable)
1032         throws IOException, RuntimeException {
1033       try {
1034         callable.instantiateServer(false);
1035         return callable.call();
1036       } catch (Throwable t) {
1037         Throwable t2 = translateException(t);
1038         if (t2 instanceof IOException) {
1039           throw (IOException)t2;
1040         } else {
1041           throw new RuntimeException(t2);
1042         }
1043       }
1044     }
1045 
1046     void close(boolean stopProxy) {
1047       if (master != null) {
1048         if (stopProxy) {
1049           HBaseRPC.stopProxy(master);
1050         }
1051         master = null;
1052         masterChecked = false;
1053       }
1054       if (stopProxy) {
1055         for (HRegionInterface i: servers.values()) {
1056           HBaseRPC.stopProxy(i);
1057         }
1058       }
1059       if (this.zooKeeper != null) {
1060         LOG.info("Closed zookeeper sessionid=0x" +
1061           Long.toHexString(this.zooKeeper.getZooKeeper().getSessionId()));
1062         this.zooKeeper.close();
1063         this.zooKeeper = null;
1064       }
1065       this.closed = true;
1066     }
1067 
1068     private Callable<MultiResponse> createCallable(
1069         final HServerAddress address,
1070         final MultiAction multi,
1071         final byte [] tableName) {
1072       final HConnection connection = this;
1073       return new Callable<MultiResponse>() {
1074         public MultiResponse call() throws IOException {
1075           return getRegionServerWithoutRetries(
1076               new ServerCallable<MultiResponse>(connection, tableName, null) {
1077                 public MultiResponse call() throws IOException {
1078                   return server.multi(multi);
1079                 }
1080                 @Override
1081                 public void instantiateServer(boolean reload) throws IOException {
1082                   server = connection.getHRegionConnection(address);
1083                 }
1084               }
1085           );
1086         }
1087       };
1088     }
1089 
1090     public void processBatch(List<Row> list,
1091         final byte[] tableName,
1092         ExecutorService pool,
1093         Object[] results) throws IOException, InterruptedException {
1094 
1095       // results must be the same size as list
1096       if (results.length != list.size()) {
1097         throw new IllegalArgumentException("argument results must be the same size as argument list");
1098       }
1099 
1100       if (list.size() == 0) {
1101         return;
1102       }
1103 
1104       // Keep track of the most recent servers for any given item for better
1105       // exceptional reporting.
1106       HServerAddress [] lastServers = new HServerAddress[results.length];
1107       List<Row> workingList = new ArrayList<Row>(list);
1108       boolean retry = true;
1109       Throwable singleRowCause = null;
1110 
1111       for (int tries = 0; tries < numRetries && retry; ++tries) {
1112 
1113         // sleep first, if this is a retry
1114         if (tries >= 1) {
1115           long sleepTime = getPauseTime(tries);
1116           LOG.debug("Retry " +tries+ ", sleep for " +sleepTime+ "ms!");
1117           Thread.sleep(sleepTime);
1118         }
1119 
1120         // step 1: break up into regionserver-sized chunks and build the data structs
1121 
1122         Map<HServerAddress, MultiAction> actionsByServer = new HashMap<HServerAddress, MultiAction>();
1123         for (int i = 0; i < workingList.size(); i++) {
1124           Row row = workingList.get(i);
1125           if (row != null) {
1126             HRegionLocation loc = locateRegion(tableName, row.getRow(), true);
1127             HServerAddress address = loc.getServerAddress();
1128             byte[] regionName = loc.getRegionInfo().getRegionName();
1129 
1130             MultiAction actions = actionsByServer.get(address);
1131             if (actions == null) {
1132               actions = new MultiAction();
1133               actionsByServer.put(address, actions);
1134             }
1135 
1136             Action action = new Action(regionName, row, i);
1137             lastServers[i] = address;
1138             actions.add(regionName, action);
1139           }
1140         }
1141 
1142         // step 2: make the requests
1143 
1144         Map<HServerAddress,Future<MultiResponse>> futures =
1145             new HashMap<HServerAddress, Future<MultiResponse>>(actionsByServer.size());
1146 
1147         for (Entry<HServerAddress, MultiAction> e : actionsByServer.entrySet()) {
1148           futures.put(e.getKey(), pool.submit(createCallable(e.getKey(), e.getValue(), tableName)));
1149         }
1150 
1151         // step 3: collect the failures and successes and prepare for retry
1152 
1153         for (Entry<HServerAddress, Future<MultiResponse>> responsePerServer : futures.entrySet()) {
1154           HServerAddress address = responsePerServer.getKey();
1155 
1156           try {
1157             Future<MultiResponse> future = responsePerServer.getValue();
1158             MultiResponse resp = future.get();
1159 
1160             if (resp == null) {
1161               // Entire server failed
1162               LOG.debug("Failed all for server: " + address + ", removing from cache");
1163               continue;
1164             }
1165 
1166             for (Entry<byte[], List<Pair<Integer,Object>>> e : resp.getResults().entrySet()) {
1167               byte[] regionName = e.getKey();
1168               List<Pair<Integer, Object>> regionResults = e.getValue();
1169               for (Pair<Integer, Object> regionResult : regionResults) {
1170                 if (regionResult == null) {
1171                   // if the first/only record is 'null' the entire region failed.
1172                   LOG.debug("Failures for region: " +
1173                       Bytes.toStringBinary(regionName) +
1174                       ", removing from cache");
1175                 } else {
1176                   // Result might be an Exception, including DNRIOE
1177                   results[regionResult.getFirst()] = regionResult.getSecond();
1178                 }
1179               }
1180             }
1181           } catch (ExecutionException e) {
1182             LOG.debug("Failed all from " + address, e);
1183           }
1184         }
1185 
1186         // step 4: identify failures and prep for a retry (if applicable).
1187 
1188         // Find failures (i.e. null Result), and add them to the workingList (in
1189         // order), so they can be retried.
1190         retry = false;
1191         workingList.clear();
1192         for (int i = 0; i < results.length; i++) {
1193           // if null (fail) or instanceof Throwable && not instanceof DNRIOE
1194           // then retry that row. else dont.
1195           if (results[i] == null ||
1196               (results[i] instanceof Throwable &&
1197                   !(results[i] instanceof DoNotRetryIOException))) {
1198 
1199             retry = true;
1200 
1201             Row row = list.get(i);
1202             workingList.add(row);
1203             deleteCachedLocation(tableName, row.getRow());
1204           } else {
1205             // add null to workingList, so the order remains consistent with the original list argument.
1206             workingList.add(null);
1207           }
1208         }
1209       }
1210 
1211       if (retry) {
1212         // Simple little check for 1 item failures.
1213         if (singleRowCause != null) {
1214           throw new IOException(singleRowCause);
1215         }
1216       }
1217 
1218 
1219       List<Throwable> exceptions = new ArrayList<Throwable>();
1220       List<Row> actions = new ArrayList<Row>();
1221       List<HServerAddress> addresses = new ArrayList<HServerAddress>();
1222 
1223       for (int i = 0 ; i < results.length; i++) {
1224         if (results[i] == null || results[i] instanceof Throwable) {
1225           exceptions.add((Throwable)results[i]);
1226           actions.add(list.get(i));
1227           addresses.add(lastServers[i]);
1228         }
1229       }
1230 
1231       if (!exceptions.isEmpty()) {
1232         throw new RetriesExhaustedWithDetailsException(exceptions,
1233             actions,
1234             addresses);
1235       }
1236     }
1237 
1238     /**
1239      * @deprecated Use HConnectionManager::processBatch instead.
1240      */
1241     public void processBatchOfPuts(List<Put> list,
1242         final byte[] tableName,
1243         ExecutorService pool) throws IOException {
1244       Object[] results = new Object[list.size()];
1245       try {
1246         processBatch((List) list, tableName, pool, results);
1247       } catch (InterruptedException e) {
1248         throw new IOException(e);
1249       } finally {
1250 
1251         // mutate list so that it is empty for complete success, or contains only failed records
1252         // results are returned in the same order as the requests in list
1253         // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
1254         for (int i = results.length - 1; i>=0; i--) {
1255           if (results[i] instanceof Result) {
1256             // successful Puts are removed from the list here.
1257             list.remove(i);
1258           }
1259         }
1260       }
1261     }
1262 
1263     private Throwable translateException(Throwable t) throws IOException {
1264       if (t instanceof UndeclaredThrowableException) {
1265         t = t.getCause();
1266       }
1267       if (t instanceof RemoteException) {
1268         t = RemoteExceptionHandler.decodeRemoteException((RemoteException)t);
1269       }
1270       if (t instanceof DoNotRetryIOException) {
1271         throw (DoNotRetryIOException)t;
1272       }
1273       return t;
1274     }
1275 
1276     /*
1277      * Return the number of cached region for a table. It will only be called
1278      * from a unit test.
1279      */
1280     int getNumberOfCachedRegionLocations(final byte[] tableName) {
1281       Integer key = Bytes.mapKey(tableName);
1282       synchronized (this.cachedRegionLocations) {
1283         SoftValueSortedMap<byte[], HRegionLocation> tableLocs =
1284           this.cachedRegionLocations.get(key);
1285 
1286         if (tableLocs == null) {
1287           return 0;
1288         }
1289         return tableLocs.values().size();
1290       }
1291     }
1292 
1293     /**
1294      * Check the region cache to see whether a region is cached yet or not.
1295      * Called by unit tests.
1296      * @param tableName tableName
1297      * @param row row
1298      * @return Region cached or not.
1299      */
1300     boolean isRegionCached(final byte[] tableName, final byte[] row) {
1301       HRegionLocation location = getCachedLocation(tableName, row);
1302       return location != null;
1303     }
1304 
1305     public void setRegionCachePrefetch(final byte[] tableName,
1306         final boolean enable) {
1307       if (!enable) {
1308         regionCachePrefetchDisabledTables.add(Bytes.mapKey(tableName));
1309       }
1310       else {
1311         regionCachePrefetchDisabledTables.remove(Bytes.mapKey(tableName));
1312       }
1313     }
1314 
1315     public boolean getRegionCachePrefetch(final byte[] tableName) {
1316       return !regionCachePrefetchDisabledTables.contains(Bytes.mapKey(tableName));
1317     }
1318 
1319     public void prewarmRegionCache(final byte[] tableName,
1320         final Map<HRegionInfo, HServerAddress> regions) {
1321       for (Map.Entry<HRegionInfo, HServerAddress> e : regions.entrySet()) {
1322         cacheLocation(tableName,
1323             new HRegionLocation(e.getKey(), e.getValue()));
1324       }
1325     }
1326 
1327     @Override
1328     public void abort(final String msg, Throwable t) {
1329       if (t instanceof KeeperException.SessionExpiredException) {
1330         try {
1331           LOG.info("This client just lost it's session with ZooKeeper, trying" +
1332               " to reconnect.");
1333           resetZooKeeperTrackers();
1334           LOG.info("Reconnected successfully. This disconnect could have been" +
1335               " caused by a network partition or a long-running GC pause," +
1336               " either way it's recommended that you verify your environment.");
1337           return;
1338         } catch (ZooKeeperConnectionException e) {
1339           LOG.error("Could not reconnect to ZooKeeper after session" +
1340               " expiration, aborting");
1341           t = e;
1342         }
1343       }
1344       if (t != null) LOG.fatal(msg, t);
1345       else LOG.fatal(msg);
1346       this.closed = true;
1347     }
1348   }
1349 }