View Javadoc

1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.DataInput;
24  import java.io.DataOutput;
25  import java.io.IOException;
26  import java.lang.reflect.Proxy;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Collections;
32  import java.util.NavigableMap;
33  import java.util.TreeMap;
34  import java.util.concurrent.ExecutorService;
35  import java.util.concurrent.SynchronousQueue;
36  import java.util.concurrent.ThreadFactory;
37  import java.util.concurrent.ThreadPoolExecutor;
38  import java.util.concurrent.TimeUnit;
39  import java.util.concurrent.atomic.AtomicInteger;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.hbase.HBaseConfiguration;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HRegionLocation;
48  import org.apache.hadoop.hbase.HServerAddress;
49  import org.apache.hadoop.hbase.HTableDescriptor;
50  import org.apache.hadoop.hbase.KeyValue;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
53  import org.apache.hadoop.hbase.client.coprocessor.Batch;
54  import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
55  import org.apache.hadoop.hbase.ipc.ExecRPCInvoker;
56  import org.apache.hadoop.hbase.util.Addressing;
57  import org.apache.hadoop.hbase.util.Bytes;
58  import org.apache.hadoop.hbase.util.Pair;
59  import org.apache.hadoop.hbase.util.Threads;
60  
61  /**
62   * <p>Used to communicate with a single HBase table.
63   *
64   * <p>This class is not thread safe for reads nor write.
65   * 
66   * <p>In case of writes (Put, Delete), the underlying write buffer can
67   * be corrupted if multiple threads contend over a single HTable instance.
68   * 
69   * <p>In case of reads, some fields used by a Scan are shared among all threads.
70   * The HTable implementation can either not contract to be safe in case of a Get
71   *
72   * <p>To access a table in a multi threaded environment, please consider
73   * using the {@link HTablePool} class to create your HTable instances.
74   *
75   * <p>Instances of HTable passed the same {@link Configuration} instance will
76   * share connections to servers out on the cluster and to the zookeeper ensemble
77   * as well as caches of region locations.  This is usually a *good* thing and it
78   * is recommended to reuse the same configuration object for all your tables.
79   * This happens because they will all share the same underlying
80   * {@link HConnection} instance. See {@link HConnectionManager} for more on
81   * how this mechanism works.
82   *
83   * <p>{@link HConnection} will read most of the
84   * configuration it needs from the passed {@link Configuration} on initial
85   * construction.  Thereafter, for settings such as
86   * <code>hbase.client.pause</code>, <code>hbase.client.retries.number</code>,
87   * and <code>hbase.client.rpc.maxattempts</code> updating their values in the
88   * passed {@link Configuration} subsequent to {@link HConnection} construction
89   * will go unnoticed.  To run with changed values, make a new
90   * {@link HTable} passing a new {@link Configuration} instance that has the
91   * new configuration.
92   *
93   * <p>Note that this class implements the {@link Closeable} interface. When a
94   * HTable instance is no longer required, it *should* be closed in order to ensure
95   * that the underlying resources are promptly released. Please note that the close 
96   * method can throw java.io.IOException that must be handled.
97   *
98   * @see HBaseAdmin for create, drop, list, enable and disable of tables.
99   * @see HConnection
100  * @see HConnectionManager
101  */
102 public class HTable implements HTableInterface {
103   private static final Log LOG = LogFactory.getLog(HTable.class);
104   private HConnection connection;
105   private final byte [] tableName;
106   private volatile Configuration configuration;
107   private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
108   private long writeBufferSize;
109   private boolean clearBufferOnFail;
110   private boolean autoFlush;
111   private long currentWriteBufferSize;
112   protected int scannerCaching;
113   private int maxKeyValueSize;
114   private ExecutorService pool;  // For Multi
115   private boolean closed;
116   private int operationTimeout;
117   private final boolean cleanupPoolOnClose; // shutdown the pool in close()
118   private final boolean cleanupConnectionOnClose; // close the connection in close()
119 
120   /**
121    * Creates an object to access a HBase table.
122    * Shares zookeeper connection and other resources with other HTable instances
123    * created with the same <code>conf</code> instance.  Uses already-populated
124    * region cache if one is available, populated by any other HTable instances
125    * sharing this <code>conf</code> instance.  Recommended.
126    * @param conf Configuration object to use.
127    * @param tableName Name of the table.
128    * @throws IOException if a remote or network exception occurs
129    */
130   public HTable(Configuration conf, final String tableName)
131   throws IOException {
132     this(conf, Bytes.toBytes(tableName));
133   }
134 
135 
136   /**
137    * Creates an object to access a HBase table.
138    * Shares zookeeper connection and other resources with other HTable instances
139    * created with the same <code>conf</code> instance.  Uses already-populated
140    * region cache if one is available, populated by any other HTable instances
141    * sharing this <code>conf</code> instance.  Recommended.
142    * @param conf Configuration object to use.
143    * @param tableName Name of the table.
144    * @throws IOException if a remote or network exception occurs
145    */
146   public HTable(Configuration conf, final byte [] tableName)
147   throws IOException {
148     this.tableName = tableName;
149     this.cleanupPoolOnClose = this.cleanupConnectionOnClose = true;
150     if (conf == null) {
151       this.connection = null;
152       return;
153     }
154     this.connection = HConnectionManager.getConnection(conf);
155     this.configuration = conf;
156 
157     int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE);
158     if (maxThreads == 0) {
159       maxThreads = 1; // is there a better default?
160     }
161     long keepAliveTime = conf.getLong("hbase.htable.threads.keepalivetime", 60);
162 
163     // Using the "direct handoff" approach, new threads will only be created
164     // if it is necessary and will grow unbounded. This could be bad but in HCM
165     // we only create as many Runnables as there are region servers. It means
166     // it also scales when new region servers are added.
167     this.pool = new ThreadPoolExecutor(1, maxThreads,
168         keepAliveTime, TimeUnit.SECONDS,
169         new SynchronousQueue<Runnable>(),
170         Threads.newDaemonThreadFactory("hbase-table"));
171     ((ThreadPoolExecutor)this.pool).allowCoreThreadTimeOut(true);
172 
173     this.finishSetup();
174   }
175 
176   /**
177    * Creates an object to access a HBase table.
178    * Shares zookeeper connection and other resources with other HTable instances
179    * created with the same <code>conf</code> instance.  Uses already-populated
180    * region cache if one is available, populated by any other HTable instances
181    * sharing this <code>conf</code> instance.
182    * Use this constructor when the ExecutorService is externally managed.
183    * @param conf Configuration object to use.
184    * @param tableName Name of the table.
185    * @param pool ExecutorService to be used.
186    * @throws IOException if a remote or network exception occurs
187    */
188   public HTable(Configuration conf, final byte[] tableName, final ExecutorService pool)
189       throws IOException {
190     this.connection = HConnectionManager.getConnection(conf);
191     this.configuration = conf;
192     this.pool = pool;
193     this.tableName = tableName;
194     this.cleanupPoolOnClose = false;
195     this.cleanupConnectionOnClose = true;
196 
197     this.finishSetup();
198   }
199 
200   /**
201    * Creates an object to access a HBase table.
202    * Shares zookeeper connection and other resources with other HTable instances
203    * created with the same <code>connection</code> instance.
204    * Use this constructor when the ExecutorService and HConnection instance are
205    * externally managed.
206    * @param tableName Name of the table.
207    * @param connection HConnection to be used.
208    * @param pool ExecutorService to be used.
209    * @throws IOException if a remote or network exception occurs
210    */
211   public HTable(final byte[] tableName, final HConnection connection, 
212       final ExecutorService pool) throws IOException {
213     if (pool == null || pool.isShutdown()) {
214       throw new IllegalArgumentException("Pool is null or shut down.");
215     }
216     if (connection == null || connection.isClosed()) {
217       throw new IllegalArgumentException("Connection is null or closed.");
218     }
219     this.tableName = tableName;
220     this.cleanupPoolOnClose = this.cleanupConnectionOnClose = false;
221     this.connection = connection;
222     this.configuration = connection.getConfiguration();
223     this.pool = pool;
224 
225     this.finishSetup();
226   }
227 
228   /**
229    * setup this HTable's parameter based on the passed configuration
230    * @param conf
231    */
232   private void finishSetup() throws IOException {
233     this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
234     this.operationTimeout = HTableDescriptor.isMetaTable(tableName) ? HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT
235         : this.configuration.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
236             HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
237     this.writeBufferSize = this.configuration.getLong(
238         "hbase.client.write.buffer", 2097152);
239     this.clearBufferOnFail = true;
240     this.autoFlush = true;
241     this.currentWriteBufferSize = 0;
242     this.scannerCaching = this.configuration.getInt(
243         "hbase.client.scanner.caching", 1);
244 
245     this.maxKeyValueSize = this.configuration.getInt(
246         "hbase.client.keyvalue.maxsize", -1);
247     this.closed = false;
248   }
249 
250   /**
251    * {@inheritDoc}
252    */
253   @Override
254   public Configuration getConfiguration() {
255     return configuration;
256   }
257 
258   /**
259    * Tells whether or not a table is enabled or not. This method creates a
260    * new HBase configuration, so it might make your unit tests fail due to
261    * incorrect ZK client port.
262    * @param tableName Name of table to check.
263    * @return {@code true} if table is online.
264    * @throws IOException if a remote or network exception occurs
265 	* @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
266    */
267   @Deprecated
268   public static boolean isTableEnabled(String tableName) throws IOException {
269     return isTableEnabled(Bytes.toBytes(tableName));
270   }
271 
272   /**
273    * Tells whether or not a table is enabled or not. This method creates a
274    * new HBase configuration, so it might make your unit tests fail due to
275    * incorrect ZK client port.
276    * @param tableName Name of table to check.
277    * @return {@code true} if table is online.
278    * @throws IOException if a remote or network exception occurs
279    * @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
280    */
281   @Deprecated
282   public static boolean isTableEnabled(byte[] tableName) throws IOException {
283     return isTableEnabled(HBaseConfiguration.create(), tableName);
284   }
285 
286   /**
287    * Tells whether or not a table is enabled or not.
288    * @param conf The Configuration object to use.
289    * @param tableName Name of table to check.
290    * @return {@code true} if table is online.
291    * @throws IOException if a remote or network exception occurs
292 	* @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
293    */
294   @Deprecated
295   public static boolean isTableEnabled(Configuration conf, String tableName)
296   throws IOException {
297     return isTableEnabled(conf, Bytes.toBytes(tableName));
298   }
299 
300   /**
301    * Tells whether or not a table is enabled or not.
302    * @param conf The Configuration object to use.
303    * @param tableName Name of table to check.
304    * @return {@code true} if table is online.
305    * @throws IOException if a remote or network exception occurs
306    */
307   public static boolean isTableEnabled(Configuration conf,
308       final byte[] tableName) throws IOException {
309     return HConnectionManager.execute(new HConnectable<Boolean>(conf) {
310       @Override
311       public Boolean connect(HConnection connection) throws IOException {
312         return connection.isTableEnabled(tableName);
313       }
314     });
315   }
316 
317   /**
318    * Find region location hosting passed row using cached info
319    * @param row Row to find.
320    * @return The location of the given row.
321    * @throws IOException if a remote or network exception occurs
322    */
323   public HRegionLocation getRegionLocation(final String row)
324   throws IOException {
325     return connection.getRegionLocation(tableName, Bytes.toBytes(row), false);
326   }
327 
328   /**
329    * Finds the region on which the given row is being served.
330    * @param row Row to find.
331    * @return Location of the row.
332    * @throws IOException if a remote or network exception occurs
333    * @deprecated use {@link #getRegionLocation(byte [], boolean)} instead
334    */
335   public HRegionLocation getRegionLocation(final byte [] row)
336   throws IOException {
337     return connection.getRegionLocation(tableName, row, false);
338   }
339 
340   /**
341    * Finds the region on which the given row is being served.
342    * @param row Row to find.
343    * @param reload whether or not to reload information or just use cached
344    * information
345    * @return Location of the row.
346    * @throws IOException if a remote or network exception occurs
347    */
348   public HRegionLocation getRegionLocation(final byte [] row, boolean reload)
349   throws IOException {
350     return connection.getRegionLocation(tableName, row, reload);
351   }
352      
353   /**
354    * {@inheritDoc}
355    */
356   @Override
357   public byte [] getTableName() {
358     return this.tableName;
359   }
360 
361   /**
362    * <em>INTERNAL</em> Used by unit tests and tools to do low-level
363    * manipulations.
364    * @return An HConnection instance.
365    * @deprecated This method will be changed from public to package protected.
366    */
367   // TODO(tsuna): Remove this.  Unit tests shouldn't require public helpers.
368   public HConnection getConnection() {
369     return this.connection;
370   }
371 
372   /**
373    * Gets the number of rows that a scanner will fetch at once.
374    * <p>
375    * The default value comes from {@code hbase.client.scanner.caching}.
376    * @deprecated Use {@link Scan#setCaching(int)} and {@link Scan#getCaching()}
377    */
378   public int getScannerCaching() {
379     return scannerCaching;
380   }
381 
382   /**
383    * Sets the number of rows that a scanner will fetch at once.
384    * <p>
385    * This will override the value specified by
386    * {@code hbase.client.scanner.caching}.
387    * Increasing this value will reduce the amount of work needed each time
388    * {@code next()} is called on a scanner, at the expense of memory use
389    * (since more rows will need to be maintained in memory by the scanners).
390    * @param scannerCaching the number of rows a scanner will fetch at once.
391    * @deprecated Use {@link Scan#setCaching(int)}
392    */
393   public void setScannerCaching(int scannerCaching) {
394     this.scannerCaching = scannerCaching;
395   }
396 
397   /**
398    * {@inheritDoc}
399    */
400   @Override
401   public HTableDescriptor getTableDescriptor() throws IOException {
402     return new UnmodifyableHTableDescriptor(
403       this.connection.getHTableDescriptor(this.tableName));
404   }
405 
406   /**
407    * Gets the starting row key for every region in the currently open table.
408    * <p>
409    * This is mainly useful for the MapReduce integration.
410    * @return Array of region starting row keys
411    * @throws IOException if a remote or network exception occurs
412    */
413   public byte [][] getStartKeys() throws IOException {
414     return getStartEndKeys().getFirst();
415   }
416 
417   /**
418    * Gets the ending row key for every region in the currently open table.
419    * <p>
420    * This is mainly useful for the MapReduce integration.
421    * @return Array of region ending row keys
422    * @throws IOException if a remote or network exception occurs
423    */
424   public byte[][] getEndKeys() throws IOException {
425     return getStartEndKeys().getSecond();
426   }
427 
428   /**
429    * Gets the starting and ending row keys for every region in the currently
430    * open table.
431    * <p>
432    * This is mainly useful for the MapReduce integration.
433    * @return Pair of arrays of region starting and ending row keys
434    * @throws IOException if a remote or network exception occurs
435    */
436   public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException {
437     NavigableMap<HRegionInfo, ServerName> regions = getRegionLocations();
438     final List<byte[]> startKeyList = new ArrayList<byte[]>(regions.size());
439     final List<byte[]> endKeyList = new ArrayList<byte[]>(regions.size());
440 
441     for (HRegionInfo region : regions.keySet()) {
442       startKeyList.add(region.getStartKey());
443       endKeyList.add(region.getEndKey());
444     }
445 
446     return new Pair<byte [][], byte [][]>(
447       startKeyList.toArray(new byte[startKeyList.size()][]),
448       endKeyList.toArray(new byte[endKeyList.size()][]));
449   }
450 
451   /**
452    * Gets all the regions and their address for this table.
453    * @return A map of HRegionInfo with it's server address
454    * @throws IOException if a remote or network exception occurs
455    * @deprecated Use {@link #getRegionLocations()} or {@link #getStartEndKeys()}
456    */
457   public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
458     final Map<HRegionInfo, HServerAddress> regionMap =
459       new TreeMap<HRegionInfo, HServerAddress>();
460 
461     final Map<HRegionInfo, ServerName> regionLocations = getRegionLocations();
462 
463     for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) {
464       HServerAddress server = new HServerAddress();
465       ServerName serverName = entry.getValue();
466       if (serverName != null && serverName.getHostAndPort() != null) {
467         server = new HServerAddress(Addressing.createInetSocketAddressFromHostAndPortStr(
468             serverName.getHostAndPort()));
469       }
470       regionMap.put(entry.getKey(), server);
471     }
472 
473     return regionMap;
474   }
475 
476   /**
477    * Gets all the regions and their address for this table.
478    * <p>
479    * This is mainly useful for the MapReduce integration.
480    * @return A map of HRegionInfo with it's server address
481    * @throws IOException if a remote or network exception occurs
482    */
483   public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOException {
484     return MetaScanner.allTableRegions(getConfiguration(), getTableName(), false);
485   }
486 
487   /**
488    * Get the corresponding regions for an arbitrary range of keys.
489    * <p>
490    * @param startRow Starting row in range, inclusive
491    * @param endRow Ending row in range, exclusive
492    * @return A list of HRegionLocations corresponding to the regions that
493    * contain the specified range
494    * @throws IOException if a remote or network exception occurs
495    */
496   public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
497     final byte [] endKey) throws IOException {
498     return getRegionsInRange(startKey, endKey, false);
499   }
500 
501   /**
502    * Get the corresponding regions for an arbitrary range of keys.
503    * <p>
504    * @param startKey Starting row in range, inclusive
505    * @param endKey Ending row in range, exclusive
506    * @param reload true to reload information or false to use cached information
507    * @return A list of HRegionLocations corresponding to the regions that
508    * contain the specified range
509    * @throws IOException if a remote or network exception occurs
510    */
511   public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
512       final byte [] endKey, final boolean reload) throws IOException {
513     return getKeysAndRegionsInRange(startKey, endKey, false, reload).getSecond();
514   }
515 
516   /**
517    * Get the corresponding start keys and regions for an arbitrary range of
518    * keys.
519    * <p>
520    * @param startKey Starting row in range, inclusive
521    * @param endKey Ending row in range
522    * @param includeEndKey true if endRow is inclusive, false if exclusive
523    * @return A pair of list of start keys and list of HRegionLocations that
524    *         contain the specified range
525    * @throws IOException if a remote or network exception occurs
526    */
527   private Pair<List<byte[]>, List<HRegionLocation>> getKeysAndRegionsInRange(
528       final byte[] startKey, final byte[] endKey, final boolean includeEndKey)
529       throws IOException {
530     return getKeysAndRegionsInRange(startKey, endKey, includeEndKey, false);
531   }
532 
533   /**
534    * Get the corresponding start keys and regions for an arbitrary range of
535    * keys.
536    * <p>
537    * @param startKey Starting row in range, inclusive
538    * @param endKey Ending row in range
539    * @param includeEndKey true if endRow is inclusive, false if exclusive
540    * @param reload true to reload information or false to use cached information
541    * @return A pair of list of start keys and list of HRegionLocations that
542    *         contain the specified range
543    * @throws IOException if a remote or network exception occurs
544    */
545   private Pair<List<byte[]>, List<HRegionLocation>> getKeysAndRegionsInRange(
546       final byte[] startKey, final byte[] endKey, final boolean includeEndKey,
547       final boolean reload) throws IOException {
548     final boolean endKeyIsEndOfTable = Bytes.equals(endKey,HConstants.EMPTY_END_ROW);
549     if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) {
550       throw new IllegalArgumentException(
551         "Invalid range: " + Bytes.toStringBinary(startKey) +
552         " > " + Bytes.toStringBinary(endKey));
553     }
554     List<byte[]> keysInRange = new ArrayList<byte[]>();
555     List<HRegionLocation> regionsInRange = new ArrayList<HRegionLocation>();
556     byte[] currentKey = startKey;
557     do {
558       HRegionLocation regionLocation = getRegionLocation(currentKey, reload);
559       keysInRange.add(currentKey);
560       regionsInRange.add(regionLocation);
561       currentKey = regionLocation.getRegionInfo().getEndKey();
562     } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
563         && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0
564             || (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0)));
565     return new Pair<List<byte[]>, List<HRegionLocation>>(keysInRange,
566         regionsInRange);
567   }
568 
569   /**
570    * Save the passed region information and the table's regions
571    * cache.
572    * <p>
573    * This is mainly useful for the MapReduce integration. You can call
574    * {@link #deserializeRegionInfo deserializeRegionInfo}
575    * to deserialize regions information from a
576    * {@link DataInput}, then call this method to load them to cache.
577    *
578    * <pre>
579    * {@code
580    * HTable t1 = new HTable("foo");
581    * FileInputStream fis = new FileInputStream("regions.dat");
582    * DataInputStream dis = new DataInputStream(fis);
583    *
584    * Map<HRegionInfo, HServerAddress> hm = t1.deserializeRegionInfo(dis);
585    * t1.prewarmRegionCache(hm);
586    * }
587    * </pre>
588    * @param regionMap This piece of regions information will be loaded
589    * to region cache.
590    */
591   public void prewarmRegionCache(Map<HRegionInfo, HServerAddress> regionMap) {
592     this.connection.prewarmRegionCache(this.getTableName(), regionMap);
593   }
594 
595   /**
596    * Serialize the regions information of this table and output
597    * to <code>out</code>.
598    * <p>
599    * This is mainly useful for the MapReduce integration. A client could
600    * perform a large scan for all the regions for the table, serialize the
601    * region info to a file. MR job can ship a copy of the meta for the table in
602    * the DistributedCache.
603    * <pre>
604    * {@code
605    * FileOutputStream fos = new FileOutputStream("regions.dat");
606    * DataOutputStream dos = new DataOutputStream(fos);
607    * table.serializeRegionInfo(dos);
608    * dos.flush();
609    * dos.close();
610    * }
611    * </pre>
612    * @param out {@link DataOutput} to serialize this object into.
613    * @throws IOException if a remote or network exception occurs
614    */
615   public void serializeRegionInfo(DataOutput out) throws IOException {
616     Map<HRegionInfo, HServerAddress> allRegions = this.getRegionsInfo();
617     // first, write number of regions
618     out.writeInt(allRegions.size());
619     for (Map.Entry<HRegionInfo, HServerAddress> es : allRegions.entrySet()) {
620       es.getKey().write(out);
621       es.getValue().write(out);
622     }
623   }
624 
625   /**
626    * Read from <code>in</code> and deserialize the regions information.
627    *
628    * <p>It behaves similarly as {@link #getRegionsInfo getRegionsInfo}, except
629    * that it loads the region map from a {@link DataInput} object.
630    *
631    * <p>It is supposed to be followed immediately by  {@link
632    * #prewarmRegionCache prewarmRegionCache}.
633    *
634    * <p>
635    * Please refer to {@link #prewarmRegionCache prewarmRegionCache} for usage.
636    *
637    * @param in {@link DataInput} object.
638    * @return A map of HRegionInfo with its server address.
639    * @throws IOException if an I/O exception occurs.
640    */
641   public Map<HRegionInfo, HServerAddress> deserializeRegionInfo(DataInput in)
642   throws IOException {
643     final Map<HRegionInfo, HServerAddress> allRegions =
644       new TreeMap<HRegionInfo, HServerAddress>();
645 
646     // the first integer is expected to be the size of records
647     int regionsCount = in.readInt();
648     for (int i = 0; i < regionsCount; ++i) {
649       HRegionInfo hri = new HRegionInfo();
650       hri.readFields(in);
651       HServerAddress hsa = new HServerAddress();
652       hsa.readFields(in);
653       allRegions.put(hri, hsa);
654     }
655     return allRegions;
656   }
657 
658   /**
659    * {@inheritDoc}
660    */
661    @Override
662    public Result getRowOrBefore(final byte[] row, final byte[] family)
663    throws IOException {
664      return new ServerCallable<Result>(connection, tableName, row, operationTimeout) {
665        public Result call() throws IOException {
666          return server.getClosestRowBefore(location.getRegionInfo().getRegionName(),
667            row, family);
668        }
669      }.withRetries();
670    }
671 
672    /**
673     * {@inheritDoc}
674     */
675   @Override
676   public ResultScanner getScanner(final Scan scan) throws IOException {
677     if (scan.getCaching() <= 0) {
678       scan.setCaching(getScannerCaching());
679     }
680     return new ClientScanner(getConfiguration(), scan, getTableName(),
681         this.connection);
682   }
683 
684   /**
685    * {@inheritDoc}
686    */
687   @Override
688   public ResultScanner getScanner(byte [] family) throws IOException {
689     Scan scan = new Scan();
690     scan.addFamily(family);
691     return getScanner(scan);
692   }
693 
694   /**
695    * {@inheritDoc}
696    */
697   @Override
698   public ResultScanner getScanner(byte [] family, byte [] qualifier)
699   throws IOException {
700     Scan scan = new Scan();
701     scan.addColumn(family, qualifier);
702     return getScanner(scan);
703   }
704 
705   /**
706    * {@inheritDoc}
707    */
708   @Override
709   public Result get(final Get get) throws IOException {
710     return new ServerCallable<Result>(connection, tableName, get.getRow(), operationTimeout) {
711           public Result call() throws IOException {
712             return server.get(location.getRegionInfo().getRegionName(), get);
713           }
714         }.withRetries();
715   }
716 
717   /**
718    * {@inheritDoc}
719    */
720   @Override
721   public Result[] get(List<Get> gets) throws IOException {
722     if (gets.size() == 1) {
723       return new Result[]{get(gets.get(0))};
724     }
725     try {
726       Object [] r1 = batch((List)gets);
727 
728       // translate.
729       Result [] results = new Result[r1.length];
730       int i=0;
731       for (Object o : r1) {
732         // batch ensures if there is a failure we get an exception instead
733         results[i++] = (Result) o;
734       }
735 
736       return results;
737     } catch (InterruptedException e) {
738       throw new IOException(e);
739     }
740   }
741 
742   /**
743    * {@inheritDoc}
744    */
745   @Override
746   public void batch(final List<?extends Row> actions, final Object[] results)
747       throws InterruptedException, IOException {
748     connection.processBatch(actions, tableName, pool, results);
749   }
750 
751   /**
752    * {@inheritDoc}
753    */
754   @Override
755   public Object[] batch(final List<? extends Row> actions) throws InterruptedException, IOException {
756     Object[] results = new Object[actions.size()];
757     connection.processBatch(actions, tableName, pool, results);
758     return results;
759   }
760 
761   /**
762    * {@inheritDoc}
763    */
764   @Override
765   public void delete(final Delete delete)
766   throws IOException {
767     new ServerCallable<Boolean>(connection, tableName, delete.getRow(), operationTimeout) {
768           public Boolean call() throws IOException {
769             server.delete(location.getRegionInfo().getRegionName(), delete);
770             return null; // FindBugs NP_BOOLEAN_RETURN_NULL
771           }
772         }.withRetries();
773   }
774 
775   /**
776    * {@inheritDoc}
777    */
778   @Override
779   public void delete(final List<Delete> deletes)
780   throws IOException {
781     Object[] results = new Object[deletes.size()];
782     try {
783       connection.processBatch((List) deletes, tableName, pool, results);
784     } catch (InterruptedException e) {
785       throw new IOException(e);
786     } finally {
787       // mutate list so that it is empty for complete success, or contains only failed records
788       // results are returned in the same order as the requests in list
789       // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
790       for (int i = results.length - 1; i>=0; i--) {
791         // if result is not null, it succeeded
792         if (results[i] instanceof Result) {
793           deletes.remove(i);
794         }
795       }
796     }
797   }
798 
799   /**
800    * {@inheritDoc}
801    */
802   @Override
803   public void put(final Put put) throws IOException {
804     doPut(put);
805     if (autoFlush) {
806       flushCommits();
807     }
808   }
809 
810   /**
811    * {@inheritDoc}
812    */
813   @Override
814   public void put(final List<Put> puts) throws IOException {
815     for (Put put : puts) {
816       doPut(put);
817     }
818     if (autoFlush) {
819       flushCommits();
820     }
821   }
822 
823   private void doPut(Put put) throws IOException{
824     validatePut(put);
825     writeBuffer.add(put);
826     currentWriteBufferSize += put.heapSize();
827     if (currentWriteBufferSize > writeBufferSize) {
828       flushCommits();
829     }
830   }
831 
832   /**
833    * {@inheritDoc}
834    */
835   @Override
836   public void mutateRow(final RowMutations rm) throws IOException {
837     new ServerCallable<Void>(connection, tableName, rm.getRow(),
838         operationTimeout) {
839       public Void call() throws IOException {
840         server.mutateRow(location.getRegionInfo().getRegionName(), rm);
841         return null;
842       }
843     }.withRetries();
844   }
845 
846   /**
847    * {@inheritDoc}
848    */
849   @Override
850   public Result append(final Append append) throws IOException {
851     if (append.numFamilies() == 0) {
852       throw new IOException(
853           "Invalid arguments to append, no columns specified");
854     }
855     return new ServerCallable<Result>(connection, tableName, append.getRow(), operationTimeout) {
856           public Result call() throws IOException {
857             return server.append(
858                 location.getRegionInfo().getRegionName(), append);
859           }
860         }.withRetries();
861   }
862 
863   /**
864    * {@inheritDoc}
865    */
866   @Override
867   public Result increment(final Increment increment) throws IOException {
868     if (!increment.hasFamilies()) {
869       throw new IOException(
870           "Invalid arguments to increment, no columns specified");
871     }
872     return new ServerCallable<Result>(connection, tableName, increment.getRow(), operationTimeout) {
873           public Result call() throws IOException {
874             return server.increment(
875                 location.getRegionInfo().getRegionName(), increment);
876           }
877         }.withRetries();
878   }
879 
880   /**
881    * {@inheritDoc}
882    */
883   @Override
884   public long incrementColumnValue(final byte [] row, final byte [] family,
885       final byte [] qualifier, final long amount)
886   throws IOException {
887     return incrementColumnValue(row, family, qualifier, amount, true);
888   }
889 
890   /**
891    * {@inheritDoc}
892    */
893   @Override
894   public long incrementColumnValue(final byte [] row, final byte [] family,
895       final byte [] qualifier, final long amount, final boolean writeToWAL)
896   throws IOException {
897     NullPointerException npe = null;
898     if (row == null) {
899       npe = new NullPointerException("row is null");
900     } else if (family == null) {
901       npe = new NullPointerException("column is null");
902     }
903     if (npe != null) {
904       throw new IOException(
905           "Invalid arguments to incrementColumnValue", npe);
906     }
907     return new ServerCallable<Long>(connection, tableName, row, operationTimeout) {
908           public Long call() throws IOException {
909             return server.incrementColumnValue(
910                 location.getRegionInfo().getRegionName(), row, family,
911                 qualifier, amount, writeToWAL);
912           }
913         }.withRetries();
914   }
915 
916   /**
917    * {@inheritDoc}
918    */
919   @Override
920   public boolean checkAndPut(final byte [] row,
921       final byte [] family, final byte [] qualifier, final byte [] value,
922       final Put put)
923   throws IOException {
924     return new ServerCallable<Boolean>(connection, tableName, row, operationTimeout) {
925           public Boolean call() throws IOException {
926             return server.checkAndPut(location.getRegionInfo().getRegionName(),
927                 row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE;
928           }
929         }.withRetries();
930   }
931 
932 
933   /**
934    * {@inheritDoc}
935    */
936   @Override
937   public boolean checkAndDelete(final byte [] row,
938       final byte [] family, final byte [] qualifier, final byte [] value,
939       final Delete delete)
940   throws IOException {
941     return new ServerCallable<Boolean>(connection, tableName, row, operationTimeout) {
942           public Boolean call() throws IOException {
943             return server.checkAndDelete(
944                 location.getRegionInfo().getRegionName(),
945                 row, family, qualifier, value, delete)
946             ? Boolean.TRUE : Boolean.FALSE;
947           }
948         }.withRetries();
949   }
950 
951   /**
952    * {@inheritDoc}
953    */
954   @Override
955   public boolean exists(final Get get) throws IOException {
956     return new ServerCallable<Boolean>(connection, tableName, get.getRow(), operationTimeout) {
957           public Boolean call() throws IOException {
958             return server.
959                 exists(location.getRegionInfo().getRegionName(), get);
960           }
961         }.withRetries();
962   }
963 
964   /**
965    * {@inheritDoc}
966    */
967   @Override
968   public void flushCommits() throws IOException {
969     try {
970       Object[] results = new Object[writeBuffer.size()];
971       try {
972         this.connection.processBatch(writeBuffer, tableName, pool, results);
973       } catch (InterruptedException e) {
974         throw new IOException(e);
975       } finally {
976         // mutate list so that it is empty for complete success, or contains
977         // only failed records results are returned in the same order as the
978         // requests in list walk the list backwards, so we can remove from list
979         // without impacting the indexes of earlier members
980         for (int i = results.length - 1; i>=0; i--) {
981           if (results[i] instanceof Result) {
982             // successful Puts are removed from the list here.
983             writeBuffer.remove(i);
984           }
985         }
986       }
987     } finally {
988       if (clearBufferOnFail) {
989         writeBuffer.clear();
990         currentWriteBufferSize = 0;
991       } else {
992         // the write buffer was adjusted by processBatchOfPuts
993         currentWriteBufferSize = 0;
994         for (Put aPut : writeBuffer) {
995           currentWriteBufferSize += aPut.heapSize();
996         }
997       }
998     }
999   }
1000 
1001   /**
1002    * {@inheritDoc}
1003    */
1004   @Override
1005   public void close() throws IOException {
1006     if (this.closed) {
1007       return;
1008     }
1009     flushCommits();
1010     if (cleanupPoolOnClose) {
1011       this.pool.shutdown();
1012     }
1013     if (cleanupConnectionOnClose) {
1014       if (this.connection != null) {
1015         this.connection.close();
1016       }
1017     }
1018     this.closed = true;
1019   }
1020 
1021   // validate for well-formedness
1022   private void validatePut(final Put put) throws IllegalArgumentException{
1023     if (put.isEmpty()) {
1024       throw new IllegalArgumentException("No columns to insert");
1025     }
1026     if (maxKeyValueSize > 0) {
1027       for (List<KeyValue> list : put.getFamilyMap().values()) {
1028         for (KeyValue kv : list) {
1029           if (kv.getLength() > maxKeyValueSize) {
1030             throw new IllegalArgumentException("KeyValue size too large");
1031           }
1032         }
1033       }
1034     }
1035   }
1036 
1037   /**
1038    * {@inheritDoc}
1039    */
1040   @Override
1041   public RowLock lockRow(final byte [] row)
1042   throws IOException {
1043     return new ServerCallable<RowLock>(connection, tableName, row, operationTimeout) {
1044         public RowLock call() throws IOException {
1045           long lockId =
1046               server.lockRow(location.getRegionInfo().getRegionName(), row);
1047           return new RowLock(row,lockId);
1048         }
1049       }.withRetries();
1050   }
1051 
1052   /**
1053    * {@inheritDoc}
1054    */
1055   @Override
1056   public void unlockRow(final RowLock rl)
1057   throws IOException {
1058     new ServerCallable<Boolean>(connection, tableName, rl.getRow(), operationTimeout) {
1059         public Boolean call() throws IOException {
1060           server.unlockRow(location.getRegionInfo().getRegionName(),
1061               rl.getLockId());
1062           return null; // FindBugs NP_BOOLEAN_RETURN_NULL
1063         }
1064       }.withRetries();
1065   }
1066 
1067   /**
1068    * {@inheritDoc}
1069    */
1070   @Override
1071   public boolean isAutoFlush() {
1072     return autoFlush;
1073   }
1074 
1075   /**
1076    * See {@link #setAutoFlush(boolean, boolean)}
1077    *
1078    * @param autoFlush
1079    *          Whether or not to enable 'auto-flush'.
1080    */
1081   public void setAutoFlush(boolean autoFlush) {
1082     setAutoFlush(autoFlush, autoFlush);
1083   }
1084 
1085   /**
1086    * Turns 'auto-flush' on or off.
1087    * <p>
1088    * When enabled (default), {@link Put} operations don't get buffered/delayed
1089    * and are immediately executed. Failed operations are not retried. This is
1090    * slower but safer.
1091    * <p>
1092    * Turning off {@link #autoFlush} means that multiple {@link Put}s will be
1093    * accepted before any RPC is actually sent to do the write operations. If the
1094    * application dies before pending writes get flushed to HBase, data will be
1095    * lost.
1096    * <p>
1097    * When you turn {@link #autoFlush} off, you should also consider the
1098    * {@link #clearBufferOnFail} option. By default, asynchronous {@link Put}
1099    * requests will be retried on failure until successful. However, this can
1100    * pollute the writeBuffer and slow down batching performance. Additionally,
1101    * you may want to issue a number of Put requests and call
1102    * {@link #flushCommits()} as a barrier. In both use cases, consider setting
1103    * clearBufferOnFail to true to erase the buffer after {@link #flushCommits()}
1104    * has been called, regardless of success.
1105    *
1106    * @param autoFlush
1107    *          Whether or not to enable 'auto-flush'.
1108    * @param clearBufferOnFail
1109    *          Whether to keep Put failures in the writeBuffer
1110    * @see #flushCommits
1111    */
1112   public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
1113     this.autoFlush = autoFlush;
1114     this.clearBufferOnFail = autoFlush || clearBufferOnFail;
1115   }
1116 
1117   /**
1118    * Returns the maximum size in bytes of the write buffer for this HTable.
1119    * <p>
1120    * The default value comes from the configuration parameter
1121    * {@code hbase.client.write.buffer}.
1122    * @return The size of the write buffer in bytes.
1123    */
1124   public long getWriteBufferSize() {
1125     return writeBufferSize;
1126   }
1127 
1128   /**
1129    * Sets the size of the buffer in bytes.
1130    * <p>
1131    * If the new size is less than the current amount of data in the
1132    * write buffer, the buffer gets flushed.
1133    * @param writeBufferSize The new write buffer size, in bytes.
1134    * @throws IOException if a remote or network exception occurs.
1135    */
1136   public void setWriteBufferSize(long writeBufferSize) throws IOException {
1137     this.writeBufferSize = writeBufferSize;
1138     if(currentWriteBufferSize > writeBufferSize) {
1139       flushCommits();
1140     }
1141   }
1142 
1143   /**
1144    * Returns the write buffer.
1145    * @return The current write buffer.
1146    */
1147   public ArrayList<Put> getWriteBuffer() {
1148     return writeBuffer;
1149   }
1150 
1151   /**
1152    * The pool is used for mutli requests for this HTable
1153    * @return the pool used for mutli
1154    */
1155   ExecutorService getPool() {
1156     return this.pool;
1157   }
1158 
1159   /**
1160    * Enable or disable region cache prefetch for the table. It will be
1161    * applied for the given table's all HTable instances who share the same
1162    * connection. By default, the cache prefetch is enabled.
1163    * @param tableName name of table to configure.
1164    * @param enable Set to true to enable region cache prefetch. Or set to
1165    * false to disable it.
1166    * @throws IOException
1167    */
1168   public static void setRegionCachePrefetch(final byte[] tableName,
1169       final boolean enable) throws IOException {
1170     HConnectionManager.execute(new HConnectable<Void>(HBaseConfiguration
1171         .create()) {
1172       @Override
1173       public Void connect(HConnection connection) throws IOException {
1174         connection.setRegionCachePrefetch(tableName, enable);
1175         return null;
1176       }
1177     });
1178   }
1179 
1180   /**
1181    * Enable or disable region cache prefetch for the table. It will be
1182    * applied for the given table's all HTable instances who share the same
1183    * connection. By default, the cache prefetch is enabled.
1184    * @param conf The Configuration object to use.
1185    * @param tableName name of table to configure.
1186    * @param enable Set to true to enable region cache prefetch. Or set to
1187    * false to disable it.
1188    * @throws IOException
1189    */
1190   public static void setRegionCachePrefetch(final Configuration conf,
1191       final byte[] tableName, final boolean enable) throws IOException {
1192     HConnectionManager.execute(new HConnectable<Void>(conf) {
1193       @Override
1194       public Void connect(HConnection connection) throws IOException {
1195         connection.setRegionCachePrefetch(tableName, enable);
1196         return null;
1197       }
1198     });
1199   }
1200 
1201   /**
1202    * Check whether region cache prefetch is enabled or not for the table.
1203    * @param conf The Configuration object to use.
1204    * @param tableName name of table to check
1205    * @return true if table's region cache prefecth is enabled. Otherwise
1206    * it is disabled.
1207    * @throws IOException
1208    */
1209   public static boolean getRegionCachePrefetch(final Configuration conf,
1210       final byte[] tableName) throws IOException {
1211     return HConnectionManager.execute(new HConnectable<Boolean>(conf) {
1212       @Override
1213       public Boolean connect(HConnection connection) throws IOException {
1214         return connection.getRegionCachePrefetch(tableName);
1215       }
1216     });
1217   }
1218 
1219   /**
1220    * Check whether region cache prefetch is enabled or not for the table.
1221    * @param tableName name of table to check
1222    * @return true if table's region cache prefecth is enabled. Otherwise
1223    * it is disabled.
1224    * @throws IOException
1225    */
1226   public static boolean getRegionCachePrefetch(final byte[] tableName) throws IOException {
1227     return HConnectionManager.execute(new HConnectable<Boolean>(
1228         HBaseConfiguration.create()) {
1229       @Override
1230       public Boolean connect(HConnection connection) throws IOException {
1231         return connection.getRegionCachePrefetch(tableName);
1232       }
1233     });
1234  }
1235 
1236   /**
1237    * Explicitly clears the region cache to fetch the latest value from META.
1238    * This is a power user function: avoid unless you know the ramifications.
1239    */
1240   public void clearRegionCache() {
1241     this.connection.clearRegionCache();
1242   }
1243 
1244   /**
1245    * {@inheritDoc}
1246    */
1247   @Override
1248   public <T extends CoprocessorProtocol> T coprocessorProxy(
1249       Class<T> protocol, byte[] row) {
1250     return (T)Proxy.newProxyInstance(this.getClass().getClassLoader(),
1251         new Class[]{protocol},
1252         new ExecRPCInvoker(configuration,
1253             connection,
1254             protocol,
1255             tableName,
1256             row));
1257   }
1258 
1259   /**
1260    * {@inheritDoc}
1261    */
1262   @Override
1263   public <T extends CoprocessorProtocol, R> Map<byte[],R> coprocessorExec(
1264       Class<T> protocol, byte[] startKey, byte[] endKey,
1265       Batch.Call<T,R> callable)
1266       throws IOException, Throwable {
1267 
1268     final Map<byte[],R> results =  Collections.synchronizedMap(new TreeMap<byte[],R>(
1269         Bytes.BYTES_COMPARATOR));
1270     coprocessorExec(protocol, startKey, endKey, callable,
1271         new Batch.Callback<R>(){
1272       public void update(byte[] region, byte[] row, R value) {
1273         results.put(region, value);
1274       }
1275     });
1276     return results;
1277   }
1278 
1279   /**
1280    * {@inheritDoc}
1281    */
1282   @Override
1283   public <T extends CoprocessorProtocol, R> void coprocessorExec(
1284       Class<T> protocol, byte[] startKey, byte[] endKey,
1285       Batch.Call<T,R> callable, Batch.Callback<R> callback)
1286       throws IOException, Throwable {
1287 
1288     // get regions covered by the row range
1289     List<byte[]> keys = getStartKeysInRange(startKey, endKey);
1290     connection.processExecs(protocol, keys, tableName, pool, callable,
1291         callback);
1292   }
1293 
1294   private List<byte[]> getStartKeysInRange(byte[] start, byte[] end)
1295   throws IOException {
1296     if (start == null) {
1297       start = HConstants.EMPTY_START_ROW;
1298     }
1299     if (end == null) {
1300       end = HConstants.EMPTY_END_ROW;
1301     }
1302     return getKeysAndRegionsInRange(start, end, true).getFirst();
1303   }
1304 
1305   public void setOperationTimeout(int operationTimeout) {
1306     this.operationTimeout = operationTimeout;
1307   }
1308 
1309   public int getOperationTimeout() {
1310     return operationTimeout;
1311   }
1312 
1313 }