View Javadoc

1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.DataInput;
24  import java.io.DataOutput;
25  import java.io.IOException;
26  import java.lang.reflect.Proxy;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Collections;
32  import java.util.NavigableMap;
33  import java.util.TreeMap;
34  import java.util.concurrent.ExecutorService;
35  import java.util.concurrent.SynchronousQueue;
36  import java.util.concurrent.ThreadFactory;
37  import java.util.concurrent.ThreadPoolExecutor;
38  import java.util.concurrent.TimeUnit;
39  import java.util.concurrent.atomic.AtomicInteger;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.hbase.HBaseConfiguration;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HRegionLocation;
48  import org.apache.hadoop.hbase.HServerAddress;
49  import org.apache.hadoop.hbase.HTableDescriptor;
50  import org.apache.hadoop.hbase.KeyValue;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
53  import org.apache.hadoop.hbase.client.coprocessor.Batch;
54  import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
55  import org.apache.hadoop.hbase.ipc.ExecRPCInvoker;
56  import org.apache.hadoop.hbase.util.Addressing;
57  import org.apache.hadoop.hbase.util.Bytes;
58  import org.apache.hadoop.hbase.util.Pair;
59  import org.apache.hadoop.hbase.util.Threads;
60  
61  /**
62   * <p>Used to communicate with a single HBase table.
63   *
64   * <p>This class is not thread safe for reads nor write.
65   * 
66   * <p>In case of writes (Put, Delete), the underlying write buffer can
67   * be corrupted if multiple threads contend over a single HTable instance.
68   * 
69   * <p>In case of reads, some fields used by a Scan are shared among all threads.
70   * The HTable implementation can either not contract to be safe in case of a Get
71   *
72   * <p>To access a table in a multi threaded environment, please consider
73   * using the {@link HTablePool} class to create your HTable instances.
74   *
75   * <p>Instances of HTable passed the same {@link Configuration} instance will
76   * share connections to servers out on the cluster and to the zookeeper ensemble
77   * as well as caches of region locations.  This is usually a *good* thing and it
78   * is recommended to reuse the same configuration object for all your tables.
79   * This happens because they will all share the same underlying
80   * {@link HConnection} instance. See {@link HConnectionManager} for more on
81   * how this mechanism works.
82   *
83   * <p>{@link HConnection} will read most of the
84   * configuration it needs from the passed {@link Configuration} on initial
85   * construction.  Thereafter, for settings such as
86   * <code>hbase.client.pause</code>, <code>hbase.client.retries.number</code>,
87   * and <code>hbase.client.rpc.maxattempts</code> updating their values in the
88   * passed {@link Configuration} subsequent to {@link HConnection} construction
89   * will go unnoticed.  To run with changed values, make a new
90   * {@link HTable} passing a new {@link Configuration} instance that has the
91   * new configuration.
92   *
93   * <p>Note that this class implements the {@link Closeable} interface. When a
94   * HTable instance is no longer required, it *should* be closed in order to ensure
95   * that the underlying resources are promptly released. Please note that the close 
96   * method can throw java.io.IOException that must be handled.
97   *
98   * @see HBaseAdmin for create, drop, list, enable and disable of tables.
99   * @see HConnection
100  * @see HConnectionManager
101  */
102 public class HTable implements HTableInterface {
103   private static final Log LOG = LogFactory.getLog(HTable.class);
104   private HConnection connection;
105   private final byte [] tableName;
106   private volatile Configuration configuration;
107   private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
108   private long writeBufferSize;
109   private boolean clearBufferOnFail;
110   private boolean autoFlush;
111   private long currentWriteBufferSize;
112   protected int scannerCaching;
113   private int maxKeyValueSize;
114   private ExecutorService pool;  // For Multi
115   private boolean closed;
116   private int operationTimeout;
117   private static final int DOPUT_WB_CHECK = 10;    // i.e., doPut checks the writebuffer every X Puts.
118   private final boolean cleanupPoolOnClose; // shutdown the pool in close()
119   private final boolean cleanupConnectionOnClose; // close the connection in close()
120 
121   /**
122    * Creates an object to access a HBase table.
123    * Shares zookeeper connection and other resources with other HTable instances
124    * created with the same <code>conf</code> instance.  Uses already-populated
125    * region cache if one is available, populated by any other HTable instances
126    * sharing this <code>conf</code> instance.  Recommended.
127    * @param conf Configuration object to use.
128    * @param tableName Name of the table.
129    * @throws IOException if a remote or network exception occurs
130    */
131   public HTable(Configuration conf, final String tableName)
132   throws IOException {
133     this(conf, Bytes.toBytes(tableName));
134   }
135 
136 
137   /**
138    * Creates an object to access a HBase table.
139    * Shares zookeeper connection and other resources with other HTable instances
140    * created with the same <code>conf</code> instance.  Uses already-populated
141    * region cache if one is available, populated by any other HTable instances
142    * sharing this <code>conf</code> instance.  Recommended.
143    * @param conf Configuration object to use.
144    * @param tableName Name of the table.
145    * @throws IOException if a remote or network exception occurs
146    */
147   public HTable(Configuration conf, final byte [] tableName)
148   throws IOException {
149     this.tableName = tableName;
150     this.cleanupPoolOnClose = this.cleanupConnectionOnClose = true;
151     if (conf == null) {
152       this.connection = null;
153       return;
154     }
155     this.connection = HConnectionManager.getConnection(conf);
156     this.configuration = conf;
157 
158     int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE);
159     if (maxThreads == 0) {
160       maxThreads = 1; // is there a better default?
161     }
162     long keepAliveTime = conf.getLong("hbase.htable.threads.keepalivetime", 60);
163 
164     // Using the "direct handoff" approach, new threads will only be created
165     // if it is necessary and will grow unbounded. This could be bad but in HCM
166     // we only create as many Runnables as there are region servers. It means
167     // it also scales when new region servers are added.
168     this.pool = new ThreadPoolExecutor(1, maxThreads,
169         keepAliveTime, TimeUnit.SECONDS,
170         new SynchronousQueue<Runnable>(),
171         Threads.newDaemonThreadFactory("hbase-table"));
172     ((ThreadPoolExecutor)this.pool).allowCoreThreadTimeOut(true);
173 
174     this.finishSetup();
175   }
176 
177   /**
178    * Creates an object to access a HBase table.
179    * Shares zookeeper connection and other resources with other HTable instances
180    * created with the same <code>conf</code> instance.  Uses already-populated
181    * region cache if one is available, populated by any other HTable instances
182    * sharing this <code>conf</code> instance.
183    * Use this constructor when the ExecutorService is externally managed.
184    * @param conf Configuration object to use.
185    * @param tableName Name of the table.
186    * @param pool ExecutorService to be used.
187    * @throws IOException if a remote or network exception occurs
188    */
189   public HTable(Configuration conf, final byte[] tableName, final ExecutorService pool)
190       throws IOException {
191     this.connection = HConnectionManager.getConnection(conf);
192     this.configuration = conf;
193     this.pool = pool;
194     this.tableName = tableName;
195     this.cleanupPoolOnClose = false;
196     this.cleanupConnectionOnClose = true;
197 
198     this.finishSetup();
199   }
200 
201   /**
202    * Creates an object to access a HBase table.
203    * Shares zookeeper connection and other resources with other HTable instances
204    * created with the same <code>connection</code> instance.
205    * Use this constructor when the ExecutorService and HConnection instance are
206    * externally managed.
207    * @param tableName Name of the table.
208    * @param connection HConnection to be used.
209    * @param pool ExecutorService to be used.
210    * @throws IOException if a remote or network exception occurs
211    */
212   public HTable(final byte[] tableName, final HConnection connection, 
213       final ExecutorService pool) throws IOException {
214     if (pool == null || pool.isShutdown()) {
215       throw new IllegalArgumentException("Pool is null or shut down.");
216     }
217     if (connection == null || connection.isClosed()) {
218       throw new IllegalArgumentException("Connection is null or closed.");
219     }
220     this.tableName = tableName;
221     this.cleanupPoolOnClose = this.cleanupConnectionOnClose = false;
222     this.connection = connection;
223     this.configuration = connection.getConfiguration();
224     this.pool = pool;
225 
226     this.finishSetup();
227   }
228 
229   /**
230    * setup this HTable's parameter based on the passed configuration
231    * @param conf
232    */
233   private void finishSetup() throws IOException {
234     this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
235     this.operationTimeout = HTableDescriptor.isMetaTable(tableName) ? HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT
236         : this.configuration.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
237             HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
238     this.writeBufferSize = this.configuration.getLong(
239         "hbase.client.write.buffer", 2097152);
240     this.clearBufferOnFail = true;
241     this.autoFlush = true;
242     this.currentWriteBufferSize = 0;
243     this.scannerCaching = this.configuration.getInt(
244         "hbase.client.scanner.caching", 1);
245 
246     this.maxKeyValueSize = this.configuration.getInt(
247         "hbase.client.keyvalue.maxsize", -1);
248     this.closed = false;
249   }
250 
251   /**
252    * {@inheritDoc}
253    */
254   @Override
255   public Configuration getConfiguration() {
256     return configuration;
257   }
258 
259   /**
260    * Tells whether or not a table is enabled or not. This method creates a
261    * new HBase configuration, so it might make your unit tests fail due to
262    * incorrect ZK client port.
263    * @param tableName Name of table to check.
264    * @return {@code true} if table is online.
265    * @throws IOException if a remote or network exception occurs
266 	* @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
267    */
268   @Deprecated
269   public static boolean isTableEnabled(String tableName) throws IOException {
270     return isTableEnabled(Bytes.toBytes(tableName));
271   }
272 
273   /**
274    * Tells whether or not a table is enabled or not. This method creates a
275    * new HBase configuration, so it might make your unit tests fail due to
276    * incorrect ZK client port.
277    * @param tableName Name of table to check.
278    * @return {@code true} if table is online.
279    * @throws IOException if a remote or network exception occurs
280    * @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
281    */
282   @Deprecated
283   public static boolean isTableEnabled(byte[] tableName) throws IOException {
284     return isTableEnabled(HBaseConfiguration.create(), tableName);
285   }
286 
287   /**
288    * Tells whether or not a table is enabled or not.
289    * @param conf The Configuration object to use.
290    * @param tableName Name of table to check.
291    * @return {@code true} if table is online.
292    * @throws IOException if a remote or network exception occurs
293 	* @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
294    */
295   @Deprecated
296   public static boolean isTableEnabled(Configuration conf, String tableName)
297   throws IOException {
298     return isTableEnabled(conf, Bytes.toBytes(tableName));
299   }
300 
301   /**
302    * Tells whether or not a table is enabled or not.
303    * @param conf The Configuration object to use.
304    * @param tableName Name of table to check.
305    * @return {@code true} if table is online.
306    * @throws IOException if a remote or network exception occurs
307    */
308   public static boolean isTableEnabled(Configuration conf,
309       final byte[] tableName) throws IOException {
310     return HConnectionManager.execute(new HConnectable<Boolean>(conf) {
311       @Override
312       public Boolean connect(HConnection connection) throws IOException {
313         return connection.isTableEnabled(tableName);
314       }
315     });
316   }
317 
318   /**
319    * Find region location hosting passed row using cached info
320    * @param row Row to find.
321    * @return The location of the given row.
322    * @throws IOException if a remote or network exception occurs
323    */
324   public HRegionLocation getRegionLocation(final String row)
325   throws IOException {
326     return connection.getRegionLocation(tableName, Bytes.toBytes(row), false);
327   }
328 
329   /**
330    * Finds the region on which the given row is being served.
331    * @param row Row to find.
332    * @return Location of the row.
333    * @throws IOException if a remote or network exception occurs
334    * @deprecated use {@link #getRegionLocation(byte [], boolean)} instead
335    */
336   public HRegionLocation getRegionLocation(final byte [] row)
337   throws IOException {
338     return connection.getRegionLocation(tableName, row, false);
339   }
340 
341   /**
342    * Finds the region on which the given row is being served.
343    * @param row Row to find.
344    * @param reload whether or not to reload information or just use cached
345    * information
346    * @return Location of the row.
347    * @throws IOException if a remote or network exception occurs
348    */
349   public HRegionLocation getRegionLocation(final byte [] row, boolean reload)
350   throws IOException {
351     return connection.getRegionLocation(tableName, row, reload);
352   }
353      
354   /**
355    * {@inheritDoc}
356    */
357   @Override
358   public byte [] getTableName() {
359     return this.tableName;
360   }
361 
362   /**
363    * <em>INTERNAL</em> Used by unit tests and tools to do low-level
364    * manipulations.
365    * @return An HConnection instance.
366    * @deprecated This method will be changed from public to package protected.
367    */
368   // TODO(tsuna): Remove this.  Unit tests shouldn't require public helpers.
369   public HConnection getConnection() {
370     return this.connection;
371   }
372 
373   /**
374    * Gets the number of rows that a scanner will fetch at once.
375    * <p>
376    * The default value comes from {@code hbase.client.scanner.caching}.
377    * @deprecated Use {@link Scan#setCaching(int)} and {@link Scan#getCaching()}
378    */
379   public int getScannerCaching() {
380     return scannerCaching;
381   }
382 
383   /**
384    * Sets the number of rows that a scanner will fetch at once.
385    * <p>
386    * This will override the value specified by
387    * {@code hbase.client.scanner.caching}.
388    * Increasing this value will reduce the amount of work needed each time
389    * {@code next()} is called on a scanner, at the expense of memory use
390    * (since more rows will need to be maintained in memory by the scanners).
391    * @param scannerCaching the number of rows a scanner will fetch at once.
392    * @deprecated Use {@link Scan#setCaching(int)}
393    */
394   public void setScannerCaching(int scannerCaching) {
395     this.scannerCaching = scannerCaching;
396   }
397 
398   /**
399    * {@inheritDoc}
400    */
401   @Override
402   public HTableDescriptor getTableDescriptor() throws IOException {
403     return new UnmodifyableHTableDescriptor(
404       this.connection.getHTableDescriptor(this.tableName));
405   }
406 
407   /**
408    * Gets the starting row key for every region in the currently open table.
409    * <p>
410    * This is mainly useful for the MapReduce integration.
411    * @return Array of region starting row keys
412    * @throws IOException if a remote or network exception occurs
413    */
414   public byte [][] getStartKeys() throws IOException {
415     return getStartEndKeys().getFirst();
416   }
417 
418   /**
419    * Gets the ending row key for every region in the currently open table.
420    * <p>
421    * This is mainly useful for the MapReduce integration.
422    * @return Array of region ending row keys
423    * @throws IOException if a remote or network exception occurs
424    */
425   public byte[][] getEndKeys() throws IOException {
426     return getStartEndKeys().getSecond();
427   }
428 
429   /**
430    * Gets the starting and ending row keys for every region in the currently
431    * open table.
432    * <p>
433    * This is mainly useful for the MapReduce integration.
434    * @return Pair of arrays of region starting and ending row keys
435    * @throws IOException if a remote or network exception occurs
436    */
437   public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException {
438     NavigableMap<HRegionInfo, ServerName> regions = getRegionLocations();
439     final List<byte[]> startKeyList = new ArrayList<byte[]>(regions.size());
440     final List<byte[]> endKeyList = new ArrayList<byte[]>(regions.size());
441 
442     for (HRegionInfo region : regions.keySet()) {
443       startKeyList.add(region.getStartKey());
444       endKeyList.add(region.getEndKey());
445     }
446 
447     return new Pair<byte [][], byte [][]>(
448       startKeyList.toArray(new byte[startKeyList.size()][]),
449       endKeyList.toArray(new byte[endKeyList.size()][]));
450   }
451 
452   /**
453    * Gets all the regions and their address for this table.
454    * @return A map of HRegionInfo with it's server address
455    * @throws IOException if a remote or network exception occurs
456    * @deprecated Use {@link #getRegionLocations()} or {@link #getStartEndKeys()}
457    */
458   public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
459     final Map<HRegionInfo, HServerAddress> regionMap =
460       new TreeMap<HRegionInfo, HServerAddress>();
461 
462     final Map<HRegionInfo, ServerName> regionLocations = getRegionLocations();
463 
464     for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) {
465       HServerAddress server = new HServerAddress();
466       ServerName serverName = entry.getValue();
467       if (serverName != null && serverName.getHostAndPort() != null) {
468         server = new HServerAddress(Addressing.createInetSocketAddressFromHostAndPortStr(
469             serverName.getHostAndPort()));
470       }
471       regionMap.put(entry.getKey(), server);
472     }
473 
474     return regionMap;
475   }
476 
477   /**
478    * Gets all the regions and their address for this table.
479    * <p>
480    * This is mainly useful for the MapReduce integration.
481    * @return A map of HRegionInfo with it's server address
482    * @throws IOException if a remote or network exception occurs
483    */
484   public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOException {
485     return MetaScanner.allTableRegions(getConfiguration(), getTableName(), false);
486   }
487 
488   /**
489    * Get the corresponding regions for an arbitrary range of keys.
490    * <p>
491    * @param startRow Starting row in range, inclusive
492    * @param endRow Ending row in range, exclusive
493    * @return A list of HRegionLocations corresponding to the regions that
494    * contain the specified range
495    * @throws IOException if a remote or network exception occurs
496    */
497   public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
498     final byte [] endKey) throws IOException {
499     final boolean endKeyIsEndOfTable = Bytes.equals(endKey,
500                                                     HConstants.EMPTY_END_ROW);
501     if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) {
502       throw new IllegalArgumentException(
503         "Invalid range: " + Bytes.toStringBinary(startKey) +
504         " > " + Bytes.toStringBinary(endKey));
505     }
506     final List<HRegionLocation> regionList = new ArrayList<HRegionLocation>();
507     byte [] currentKey = startKey;
508     do {
509       HRegionLocation regionLocation = getRegionLocation(currentKey, false);
510       regionList.add(regionLocation);
511       currentKey = regionLocation.getRegionInfo().getEndKey();
512     } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) &&
513              (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0));
514     return regionList;
515   }
516 
517   /**
518    * Save the passed region information and the table's regions
519    * cache.
520    * <p>
521    * This is mainly useful for the MapReduce integration. You can call
522    * {@link #deserializeRegionInfo deserializeRegionInfo}
523    * to deserialize regions information from a
524    * {@link DataInput}, then call this method to load them to cache.
525    *
526    * <pre>
527    * {@code
528    * HTable t1 = new HTable("foo");
529    * FileInputStream fis = new FileInputStream("regions.dat");
530    * DataInputStream dis = new DataInputStream(fis);
531    *
532    * Map<HRegionInfo, HServerAddress> hm = t1.deserializeRegionInfo(dis);
533    * t1.prewarmRegionCache(hm);
534    * }
535    * </pre>
536    * @param regionMap This piece of regions information will be loaded
537    * to region cache.
538    */
539   public void prewarmRegionCache(Map<HRegionInfo, HServerAddress> regionMap) {
540     this.connection.prewarmRegionCache(this.getTableName(), regionMap);
541   }
542 
543   /**
544    * Serialize the regions information of this table and output
545    * to <code>out</code>.
546    * <p>
547    * This is mainly useful for the MapReduce integration. A client could
548    * perform a large scan for all the regions for the table, serialize the
549    * region info to a file. MR job can ship a copy of the meta for the table in
550    * the DistributedCache.
551    * <pre>
552    * {@code
553    * FileOutputStream fos = new FileOutputStream("regions.dat");
554    * DataOutputStream dos = new DataOutputStream(fos);
555    * table.serializeRegionInfo(dos);
556    * dos.flush();
557    * dos.close();
558    * }
559    * </pre>
560    * @param out {@link DataOutput} to serialize this object into.
561    * @throws IOException if a remote or network exception occurs
562    */
563   public void serializeRegionInfo(DataOutput out) throws IOException {
564     Map<HRegionInfo, HServerAddress> allRegions = this.getRegionsInfo();
565     // first, write number of regions
566     out.writeInt(allRegions.size());
567     for (Map.Entry<HRegionInfo, HServerAddress> es : allRegions.entrySet()) {
568       es.getKey().write(out);
569       es.getValue().write(out);
570     }
571   }
572 
573   /**
574    * Read from <code>in</code> and deserialize the regions information.
575    *
576    * <p>It behaves similarly as {@link #getRegionsInfo getRegionsInfo}, except
577    * that it loads the region map from a {@link DataInput} object.
578    *
579    * <p>It is supposed to be followed immediately by  {@link
580    * #prewarmRegionCache prewarmRegionCache}.
581    *
582    * <p>
583    * Please refer to {@link #prewarmRegionCache prewarmRegionCache} for usage.
584    *
585    * @param in {@link DataInput} object.
586    * @return A map of HRegionInfo with its server address.
587    * @throws IOException if an I/O exception occurs.
588    */
589   public Map<HRegionInfo, HServerAddress> deserializeRegionInfo(DataInput in)
590   throws IOException {
591     final Map<HRegionInfo, HServerAddress> allRegions =
592       new TreeMap<HRegionInfo, HServerAddress>();
593 
594     // the first integer is expected to be the size of records
595     int regionsCount = in.readInt();
596     for (int i = 0; i < regionsCount; ++i) {
597       HRegionInfo hri = new HRegionInfo();
598       hri.readFields(in);
599       HServerAddress hsa = new HServerAddress();
600       hsa.readFields(in);
601       allRegions.put(hri, hsa);
602     }
603     return allRegions;
604   }
605 
606   /**
607    * {@inheritDoc}
608    */
609    @Override
610    public Result getRowOrBefore(final byte[] row, final byte[] family)
611    throws IOException {
612      return new ServerCallable<Result>(connection, tableName, row, operationTimeout) {
613        public Result call() throws IOException {
614          return server.getClosestRowBefore(location.getRegionInfo().getRegionName(),
615            row, family);
616        }
617      }.withRetries();
618    }
619 
620    /**
621     * {@inheritDoc}
622     */
623   @Override
624   public ResultScanner getScanner(final Scan scan) throws IOException {
625     if (scan.getCaching() <= 0) {
626       scan.setCaching(getScannerCaching());
627     }
628     return new ClientScanner(getConfiguration(), scan, getTableName(),
629         this.connection);
630   }
631 
632   /**
633    * {@inheritDoc}
634    */
635   @Override
636   public ResultScanner getScanner(byte [] family) throws IOException {
637     Scan scan = new Scan();
638     scan.addFamily(family);
639     return getScanner(scan);
640   }
641 
642   /**
643    * {@inheritDoc}
644    */
645   @Override
646   public ResultScanner getScanner(byte [] family, byte [] qualifier)
647   throws IOException {
648     Scan scan = new Scan();
649     scan.addColumn(family, qualifier);
650     return getScanner(scan);
651   }
652 
653   /**
654    * {@inheritDoc}
655    */
656   @Override
657   public Result get(final Get get) throws IOException {
658     return new ServerCallable<Result>(connection, tableName, get.getRow(), operationTimeout) {
659           public Result call() throws IOException {
660             return server.get(location.getRegionInfo().getRegionName(), get);
661           }
662         }.withRetries();
663   }
664 
665   /**
666    * {@inheritDoc}
667    */
668   @Override
669   public Result[] get(List<Get> gets) throws IOException {
670     try {
671       Object [] r1 = batch((List)gets);
672 
673       // translate.
674       Result [] results = new Result[r1.length];
675       int i=0;
676       for (Object o : r1) {
677         // batch ensures if there is a failure we get an exception instead
678         results[i++] = (Result) o;
679       }
680 
681       return results;
682     } catch (InterruptedException e) {
683       throw new IOException(e);
684     }
685   }
686 
687   /**
688    * {@inheritDoc}
689    */
690   @Override
691   public void batch(final List<?extends Row> actions, final Object[] results)
692       throws InterruptedException, IOException {
693     connection.processBatch(actions, tableName, pool, results);
694   }
695 
696   /**
697    * {@inheritDoc}
698    */
699   @Override
700   public Object[] batch(final List<? extends Row> actions) throws InterruptedException, IOException {
701     Object[] results = new Object[actions.size()];
702     connection.processBatch(actions, tableName, pool, results);
703     return results;
704   }
705 
706   /**
707    * {@inheritDoc}
708    */
709   @Override
710   public void delete(final Delete delete)
711   throws IOException {
712     new ServerCallable<Boolean>(connection, tableName, delete.getRow(), operationTimeout) {
713           public Boolean call() throws IOException {
714             server.delete(location.getRegionInfo().getRegionName(), delete);
715             return null; // FindBugs NP_BOOLEAN_RETURN_NULL
716           }
717         }.withRetries();
718   }
719 
720   /**
721    * {@inheritDoc}
722    */
723   @Override
724   public void delete(final List<Delete> deletes)
725   throws IOException {
726     Object[] results = new Object[deletes.size()];
727     try {
728       connection.processBatch((List) deletes, tableName, pool, results);
729     } catch (InterruptedException e) {
730       throw new IOException(e);
731     } finally {
732       // mutate list so that it is empty for complete success, or contains only failed records
733       // results are returned in the same order as the requests in list
734       // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
735       for (int i = results.length - 1; i>=0; i--) {
736         // if result is not null, it succeeded
737         if (results[i] instanceof Result) {
738           deletes.remove(i);
739         }
740       }
741     }
742   }
743 
744   /**
745    * {@inheritDoc}
746    */
747   @Override
748   public void put(final Put put) throws IOException {
749     doPut(Arrays.asList(put));
750   }
751 
752   /**
753    * {@inheritDoc}
754    */
755   @Override
756   public void put(final List<Put> puts) throws IOException {
757     doPut(puts);
758   }
759 
760   private void doPut(final List<Put> puts) throws IOException {
761     int n = 0;
762     for (Put put : puts) {
763       validatePut(put);
764       writeBuffer.add(put);
765       currentWriteBufferSize += put.heapSize();
766      
767       // we need to periodically see if the writebuffer is full instead of waiting until the end of the List
768       n++;
769       if (n % DOPUT_WB_CHECK == 0 && currentWriteBufferSize > writeBufferSize) {
770         flushCommits();
771       }
772     }
773     if (autoFlush || currentWriteBufferSize > writeBufferSize) {
774       flushCommits();
775     }
776   }
777 
778   /**
779    * {@inheritDoc}
780    */
781   @Override
782   public void mutateRow(final RowMutations rm) throws IOException {
783     new ServerCallable<Void>(connection, tableName, rm.getRow(),
784         operationTimeout) {
785       public Void call() throws IOException {
786         server.mutateRow(location.getRegionInfo().getRegionName(), rm);
787         return null;
788       }
789     }.withRetries();
790   }
791 
792   /**
793    * {@inheritDoc}
794    */
795   @Override
796   public Result append(final Append append) throws IOException {
797     if (append.numFamilies() == 0) {
798       throw new IOException(
799           "Invalid arguments to append, no columns specified");
800     }
801     return new ServerCallable<Result>(connection, tableName, append.getRow(), operationTimeout) {
802           public Result call() throws IOException {
803             return server.append(
804                 location.getRegionInfo().getRegionName(), append);
805           }
806         }.withRetries();
807   }
808 
809   /**
810    * {@inheritDoc}
811    */
812   @Override
813   public Result increment(final Increment increment) throws IOException {
814     if (!increment.hasFamilies()) {
815       throw new IOException(
816           "Invalid arguments to increment, no columns specified");
817     }
818     return new ServerCallable<Result>(connection, tableName, increment.getRow(), operationTimeout) {
819           public Result call() throws IOException {
820             return server.increment(
821                 location.getRegionInfo().getRegionName(), increment);
822           }
823         }.withRetries();
824   }
825 
826   /**
827    * {@inheritDoc}
828    */
829   @Override
830   public long incrementColumnValue(final byte [] row, final byte [] family,
831       final byte [] qualifier, final long amount)
832   throws IOException {
833     return incrementColumnValue(row, family, qualifier, amount, true);
834   }
835 
836   /**
837    * {@inheritDoc}
838    */
839   @Override
840   public long incrementColumnValue(final byte [] row, final byte [] family,
841       final byte [] qualifier, final long amount, final boolean writeToWAL)
842   throws IOException {
843     NullPointerException npe = null;
844     if (row == null) {
845       npe = new NullPointerException("row is null");
846     } else if (family == null) {
847       npe = new NullPointerException("column is null");
848     }
849     if (npe != null) {
850       throw new IOException(
851           "Invalid arguments to incrementColumnValue", npe);
852     }
853     return new ServerCallable<Long>(connection, tableName, row, operationTimeout) {
854           public Long call() throws IOException {
855             return server.incrementColumnValue(
856                 location.getRegionInfo().getRegionName(), row, family,
857                 qualifier, amount, writeToWAL);
858           }
859         }.withRetries();
860   }
861 
862   /**
863    * {@inheritDoc}
864    */
865   @Override
866   public boolean checkAndPut(final byte [] row,
867       final byte [] family, final byte [] qualifier, final byte [] value,
868       final Put put)
869   throws IOException {
870     return new ServerCallable<Boolean>(connection, tableName, row, operationTimeout) {
871           public Boolean call() throws IOException {
872             return server.checkAndPut(location.getRegionInfo().getRegionName(),
873                 row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE;
874           }
875         }.withRetries();
876   }
877 
878 
879   /**
880    * {@inheritDoc}
881    */
882   @Override
883   public boolean checkAndDelete(final byte [] row,
884       final byte [] family, final byte [] qualifier, final byte [] value,
885       final Delete delete)
886   throws IOException {
887     return new ServerCallable<Boolean>(connection, tableName, row, operationTimeout) {
888           public Boolean call() throws IOException {
889             return server.checkAndDelete(
890                 location.getRegionInfo().getRegionName(),
891                 row, family, qualifier, value, delete)
892             ? Boolean.TRUE : Boolean.FALSE;
893           }
894         }.withRetries();
895   }
896 
897   /**
898    * {@inheritDoc}
899    */
900   @Override
901   public boolean exists(final Get get) throws IOException {
902     return new ServerCallable<Boolean>(connection, tableName, get.getRow(), operationTimeout) {
903           public Boolean call() throws IOException {
904             return server.
905                 exists(location.getRegionInfo().getRegionName(), get);
906           }
907         }.withRetries();
908   }
909 
910   /**
911    * {@inheritDoc}
912    */
913   @Override
914   public void flushCommits() throws IOException {
915     try {
916       Object[] results = new Object[writeBuffer.size()];
917       try {
918         this.connection.processBatch(writeBuffer, tableName, pool, results);
919       } catch (InterruptedException e) {
920         throw new IOException(e);
921       } finally {
922         // mutate list so that it is empty for complete success, or contains
923         // only failed records results are returned in the same order as the
924         // requests in list walk the list backwards, so we can remove from list
925         // without impacting the indexes of earlier members
926         for (int i = results.length - 1; i>=0; i--) {
927           if (results[i] instanceof Result) {
928             // successful Puts are removed from the list here.
929             writeBuffer.remove(i);
930           }
931         }
932       }
933     } finally {
934       if (clearBufferOnFail) {
935         writeBuffer.clear();
936         currentWriteBufferSize = 0;
937       } else {
938         // the write buffer was adjusted by processBatchOfPuts
939         currentWriteBufferSize = 0;
940         for (Put aPut : writeBuffer) {
941           currentWriteBufferSize += aPut.heapSize();
942         }
943       }
944     }
945   }
946 
947   /**
948    * {@inheritDoc}
949    */
950   @Override
951   public void close() throws IOException {
952     if (this.closed) {
953       return;
954     }
955     flushCommits();
956     if (cleanupPoolOnClose) {
957       this.pool.shutdown();
958     }
959     if (cleanupConnectionOnClose) {
960       if (this.connection != null) {
961         this.connection.close();
962       }
963     }
964     this.closed = true;
965   }
966 
967   // validate for well-formedness
968   private void validatePut(final Put put) throws IllegalArgumentException{
969     if (put.isEmpty()) {
970       throw new IllegalArgumentException("No columns to insert");
971     }
972     if (maxKeyValueSize > 0) {
973       for (List<KeyValue> list : put.getFamilyMap().values()) {
974         for (KeyValue kv : list) {
975           if (kv.getLength() > maxKeyValueSize) {
976             throw new IllegalArgumentException("KeyValue size too large");
977           }
978         }
979       }
980     }
981   }
982 
983   /**
984    * {@inheritDoc}
985    */
986   @Override
987   public RowLock lockRow(final byte [] row)
988   throws IOException {
989     return new ServerCallable<RowLock>(connection, tableName, row, operationTimeout) {
990         public RowLock call() throws IOException {
991           long lockId =
992               server.lockRow(location.getRegionInfo().getRegionName(), row);
993           return new RowLock(row,lockId);
994         }
995       }.withRetries();
996   }
997 
998   /**
999    * {@inheritDoc}
1000    */
1001   @Override
1002   public void unlockRow(final RowLock rl)
1003   throws IOException {
1004     new ServerCallable<Boolean>(connection, tableName, rl.getRow(), operationTimeout) {
1005         public Boolean call() throws IOException {
1006           server.unlockRow(location.getRegionInfo().getRegionName(),
1007               rl.getLockId());
1008           return null; // FindBugs NP_BOOLEAN_RETURN_NULL
1009         }
1010       }.withRetries();
1011   }
1012 
1013   /**
1014    * {@inheritDoc}
1015    */
1016   @Override
1017   public boolean isAutoFlush() {
1018     return autoFlush;
1019   }
1020 
1021   /**
1022    * See {@link #setAutoFlush(boolean, boolean)}
1023    *
1024    * @param autoFlush
1025    *          Whether or not to enable 'auto-flush'.
1026    */
1027   public void setAutoFlush(boolean autoFlush) {
1028     setAutoFlush(autoFlush, autoFlush);
1029   }
1030 
1031   /**
1032    * Turns 'auto-flush' on or off.
1033    * <p>
1034    * When enabled (default), {@link Put} operations don't get buffered/delayed
1035    * and are immediately executed. Failed operations are not retried. This is
1036    * slower but safer.
1037    * <p>
1038    * Turning off {@link #autoFlush} means that multiple {@link Put}s will be
1039    * accepted before any RPC is actually sent to do the write operations. If the
1040    * application dies before pending writes get flushed to HBase, data will be
1041    * lost.
1042    * <p>
1043    * When you turn {@link #autoFlush} off, you should also consider the
1044    * {@link #clearBufferOnFail} option. By default, asynchronous {@link Put}
1045    * requests will be retried on failure until successful. However, this can
1046    * pollute the writeBuffer and slow down batching performance. Additionally,
1047    * you may want to issue a number of Put requests and call
1048    * {@link #flushCommits()} as a barrier. In both use cases, consider setting
1049    * clearBufferOnFail to true to erase the buffer after {@link #flushCommits()}
1050    * has been called, regardless of success.
1051    *
1052    * @param autoFlush
1053    *          Whether or not to enable 'auto-flush'.
1054    * @param clearBufferOnFail
1055    *          Whether to keep Put failures in the writeBuffer
1056    * @see #flushCommits
1057    */
1058   public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
1059     this.autoFlush = autoFlush;
1060     this.clearBufferOnFail = autoFlush || clearBufferOnFail;
1061   }
1062 
1063   /**
1064    * Returns the maximum size in bytes of the write buffer for this HTable.
1065    * <p>
1066    * The default value comes from the configuration parameter
1067    * {@code hbase.client.write.buffer}.
1068    * @return The size of the write buffer in bytes.
1069    */
1070   public long getWriteBufferSize() {
1071     return writeBufferSize;
1072   }
1073 
1074   /**
1075    * Sets the size of the buffer in bytes.
1076    * <p>
1077    * If the new size is less than the current amount of data in the
1078    * write buffer, the buffer gets flushed.
1079    * @param writeBufferSize The new write buffer size, in bytes.
1080    * @throws IOException if a remote or network exception occurs.
1081    */
1082   public void setWriteBufferSize(long writeBufferSize) throws IOException {
1083     this.writeBufferSize = writeBufferSize;
1084     if(currentWriteBufferSize > writeBufferSize) {
1085       flushCommits();
1086     }
1087   }
1088 
1089   /**
1090    * Returns the write buffer.
1091    * @return The current write buffer.
1092    */
1093   public ArrayList<Put> getWriteBuffer() {
1094     return writeBuffer;
1095   }
1096 
1097   /**
1098    * The pool is used for mutli requests for this HTable
1099    * @return the pool used for mutli
1100    */
1101   ExecutorService getPool() {
1102     return this.pool;
1103   }
1104 
1105   /**
1106    * Enable or disable region cache prefetch for the table. It will be
1107    * applied for the given table's all HTable instances who share the same
1108    * connection. By default, the cache prefetch is enabled.
1109    * @param tableName name of table to configure.
1110    * @param enable Set to true to enable region cache prefetch. Or set to
1111    * false to disable it.
1112    * @throws IOException
1113    */
1114   public static void setRegionCachePrefetch(final byte[] tableName,
1115       final boolean enable) throws IOException {
1116     HConnectionManager.execute(new HConnectable<Void>(HBaseConfiguration
1117         .create()) {
1118       @Override
1119       public Void connect(HConnection connection) throws IOException {
1120         connection.setRegionCachePrefetch(tableName, enable);
1121         return null;
1122       }
1123     });
1124   }
1125 
1126   /**
1127    * Enable or disable region cache prefetch for the table. It will be
1128    * applied for the given table's all HTable instances who share the same
1129    * connection. By default, the cache prefetch is enabled.
1130    * @param conf The Configuration object to use.
1131    * @param tableName name of table to configure.
1132    * @param enable Set to true to enable region cache prefetch. Or set to
1133    * false to disable it.
1134    * @throws IOException
1135    */
1136   public static void setRegionCachePrefetch(final Configuration conf,
1137       final byte[] tableName, final boolean enable) throws IOException {
1138     HConnectionManager.execute(new HConnectable<Void>(conf) {
1139       @Override
1140       public Void connect(HConnection connection) throws IOException {
1141         connection.setRegionCachePrefetch(tableName, enable);
1142         return null;
1143       }
1144     });
1145   }
1146 
1147   /**
1148    * Check whether region cache prefetch is enabled or not for the table.
1149    * @param conf The Configuration object to use.
1150    * @param tableName name of table to check
1151    * @return true if table's region cache prefecth is enabled. Otherwise
1152    * it is disabled.
1153    * @throws IOException
1154    */
1155   public static boolean getRegionCachePrefetch(final Configuration conf,
1156       final byte[] tableName) throws IOException {
1157     return HConnectionManager.execute(new HConnectable<Boolean>(conf) {
1158       @Override
1159       public Boolean connect(HConnection connection) throws IOException {
1160         return connection.getRegionCachePrefetch(tableName);
1161       }
1162     });
1163   }
1164 
1165   /**
1166    * Check whether region cache prefetch is enabled or not for the table.
1167    * @param tableName name of table to check
1168    * @return true if table's region cache prefecth is enabled. Otherwise
1169    * it is disabled.
1170    * @throws IOException
1171    */
1172   public static boolean getRegionCachePrefetch(final byte[] tableName) throws IOException {
1173     return HConnectionManager.execute(new HConnectable<Boolean>(
1174         HBaseConfiguration.create()) {
1175       @Override
1176       public Boolean connect(HConnection connection) throws IOException {
1177         return connection.getRegionCachePrefetch(tableName);
1178       }
1179     });
1180  }
1181 
1182   /**
1183    * Explicitly clears the region cache to fetch the latest value from META.
1184    * This is a power user function: avoid unless you know the ramifications.
1185    */
1186   public void clearRegionCache() {
1187     this.connection.clearRegionCache();
1188   }
1189 
1190   /**
1191    * {@inheritDoc}
1192    */
1193   @Override
1194   public <T extends CoprocessorProtocol> T coprocessorProxy(
1195       Class<T> protocol, byte[] row) {
1196     return (T)Proxy.newProxyInstance(this.getClass().getClassLoader(),
1197         new Class[]{protocol},
1198         new ExecRPCInvoker(configuration,
1199             connection,
1200             protocol,
1201             tableName,
1202             row));
1203   }
1204 
1205   /**
1206    * {@inheritDoc}
1207    */
1208   @Override
1209   public <T extends CoprocessorProtocol, R> Map<byte[],R> coprocessorExec(
1210       Class<T> protocol, byte[] startKey, byte[] endKey,
1211       Batch.Call<T,R> callable)
1212       throws IOException, Throwable {
1213 
1214     final Map<byte[],R> results =  Collections.synchronizedMap(new TreeMap<byte[],R>(
1215         Bytes.BYTES_COMPARATOR));
1216     coprocessorExec(protocol, startKey, endKey, callable,
1217         new Batch.Callback<R>(){
1218       public void update(byte[] region, byte[] row, R value) {
1219         results.put(region, value);
1220       }
1221     });
1222     return results;
1223   }
1224 
1225   /**
1226    * {@inheritDoc}
1227    */
1228   @Override
1229   public <T extends CoprocessorProtocol, R> void coprocessorExec(
1230       Class<T> protocol, byte[] startKey, byte[] endKey,
1231       Batch.Call<T,R> callable, Batch.Callback<R> callback)
1232       throws IOException, Throwable {
1233 
1234     // get regions covered by the row range
1235     List<byte[]> keys = getStartKeysInRange(startKey, endKey);
1236     connection.processExecs(protocol, keys, tableName, pool, callable,
1237         callback);
1238   }
1239 
1240   private List<byte[]> getStartKeysInRange(byte[] start, byte[] end)
1241   throws IOException {
1242     Pair<byte[][],byte[][]> startEndKeys = getStartEndKeys();
1243     byte[][] startKeys = startEndKeys.getFirst();
1244     byte[][] endKeys = startEndKeys.getSecond();
1245 
1246     if (start == null) {
1247       start = HConstants.EMPTY_START_ROW;
1248     }
1249     if (end == null) {
1250       end = HConstants.EMPTY_END_ROW;
1251     }
1252 
1253     List<byte[]> rangeKeys = new ArrayList<byte[]>();
1254     for (int i=0; i<startKeys.length; i++) {
1255       if (Bytes.compareTo(start, startKeys[i]) >= 0 ) {
1256         if (Bytes.equals(endKeys[i], HConstants.EMPTY_END_ROW) ||
1257             Bytes.compareTo(start, endKeys[i]) < 0) {
1258           rangeKeys.add(start);
1259         }
1260       } else if (Bytes.equals(end, HConstants.EMPTY_END_ROW) ||
1261           Bytes.compareTo(startKeys[i], end) <= 0) {
1262         rangeKeys.add(startKeys[i]);
1263       } else {
1264         break; // past stop
1265       }
1266     }
1267 
1268     return rangeKeys;
1269   }
1270 
1271   public void setOperationTimeout(int operationTimeout) {
1272     this.operationTimeout = operationTimeout;
1273   }
1274 
1275   public int getOperationTimeout() {
1276     return operationTimeout;
1277   }
1278 
1279 }