View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.Iterator;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.TreeMap;
32  import java.util.concurrent.ExecutorService;
33  import java.util.concurrent.SynchronousQueue;
34  import java.util.concurrent.ThreadFactory;
35  import java.util.concurrent.ThreadPoolExecutor;
36  import java.util.concurrent.TimeUnit;
37  import java.util.concurrent.atomic.AtomicInteger;
38  
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.DoNotRetryIOException;
43  import org.apache.hadoop.hbase.HBaseConfiguration;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HRegionLocation;
47  import org.apache.hadoop.hbase.HServerAddress;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.KeyValue;
50  import org.apache.hadoop.hbase.NotServingRegionException;
51  import org.apache.hadoop.hbase.UnknownScannerException;
52  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
53  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
54  import org.apache.hadoop.hbase.util.Bytes;
55  import org.apache.hadoop.hbase.util.Pair;
56  import org.apache.hadoop.hbase.util.Writables;
57  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
58  import org.apache.zookeeper.KeeperException;
59  
60  /**
61   * Used to communicate with a single HBase table.
62   *
63   * This class is not thread safe for updates; the underlying write buffer can
64   * be corrupted if multiple threads contend over a single HTable instance.
65   *
66   * <p>Instances of HTable passed the same {@link Configuration} instance will
67   * share connections to servers out on the cluster and to the zookeeper ensemble
68   * as well as caches of region locations.  This is usually a *good* thing and it
69   * is recommended to reuse the same configuration object for all your tables.
70   * This happens because they will all share the same underlying
71   * {@link HConnection} instance. See {@link HConnectionManager} for more on
72   * how this mechanism works.
73   *
74   * <p>{@link HConnection} will read most of the
75   * configuration it needs from the passed {@link Configuration} on initial
76   * construction.  Thereafter, for settings such as
77   * <code>hbase.client.pause</code>, <code>hbase.client.retries.number</code>,
78   * and <code>hbase.client.rpc.maxattempts</code> updating their values in the
79   * passed {@link Configuration} subsequent to {@link HConnection} construction
80   * will go unnoticed.  To run with changed values, make a new
81   * {@link HTable} passing a new {@link Configuration} instance that has the
82   * new configuration.
83   *
84   * @see HBaseAdmin for create, drop, list, enable and disable of tables.
85   * @see HConnection
86   * @see HConnectionManager
87   */
88  public class HTable implements HTableInterface {
89    private static final Log LOG = LogFactory.getLog(HTable.class);
90    private final HConnection connection;
91    private final byte [] tableName;
92    protected final int scannerTimeout;
93    private volatile Configuration configuration;
94    private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
95    private long writeBufferSize;
96    private boolean autoFlush;
97    private long currentWriteBufferSize;
98    protected int scannerCaching;
99    private int maxKeyValueSize;
100   private ExecutorService pool;  // For Multi
101   private long maxScannerResultSize;
102 
103   /**
104    * Creates an object to access a HBase table.
105    * Internally it creates a new instance of {@link Configuration} and a new
106    * client to zookeeper as well as other resources.  It also comes up with
107    * a fresh view of the cluster and must do discovery from scratch of region
108    * locations; i.e. it will not make use of already-cached region locations if
109    * available. Use only when being quick and dirty.
110    * @throws IOException if a remote or network exception occurs
111    * @see #HTable(Configuration, String)
112    */
113   public HTable(final String tableName)
114   throws IOException {
115     this(HBaseConfiguration.create(), Bytes.toBytes(tableName));
116   }
117 
118   /**
119    * Creates an object to access a HBase table.
120    * Internally it creates a new instance of {@link Configuration} and a new
121    * client to zookeeper as well as other resources.  It also comes up with
122    * a fresh view of the cluster and must do discovery from scratch of region
123    * locations; i.e. it will not make use of already-cached region locations if
124    * available. Use only when being quick and dirty.
125    * @param tableName Name of the table.
126    * @throws IOException if a remote or network exception occurs
127    * @see #HTable(Configuration, String)
128    */
129   public HTable(final byte [] tableName)
130   throws IOException {
131     this(HBaseConfiguration.create(), tableName);
132   }
133 
134   /**
135    * Creates an object to access a HBase table.
136    * Shares zookeeper connection and other resources with other HTable instances
137    * created with the same <code>conf</code> instance.  Uses already-populated
138    * region cache if one is available, populated by any other HTable instances
139    * sharing this <code>conf</code> instance.  Recommended.
140    * @param conf Configuration object to use.
141    * @param tableName Name of the table.
142    * @throws IOException if a remote or network exception occurs
143    */
144   public HTable(Configuration conf, final String tableName)
145   throws IOException {
146     this(conf, Bytes.toBytes(tableName));
147   }
148 
149 
150   /**
151    * Creates an object to access a HBase table.
152    * Shares zookeeper connection and other resources with other HTable instances
153    * created with the same <code>conf</code> instance.  Uses already-populated
154    * region cache if one is available, populated by any other HTable instances
155    * sharing this <code>conf</code> instance.  Recommended.
156    * @param conf Configuration object to use.
157    * @param tableName Name of the table.
158    * @throws IOException if a remote or network exception occurs
159    */
160   public HTable(Configuration conf, final byte [] tableName)
161   throws IOException {
162     this.tableName = tableName;
163     if (conf == null) {
164       this.scannerTimeout = 0;
165       this.connection = null;
166       return;
167     }
168     this.connection = HConnectionManager.getConnection(conf);
169     this.scannerTimeout =
170       (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
171     this.configuration = conf;
172     this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
173     this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152);
174     this.autoFlush = true;
175     this.currentWriteBufferSize = 0;
176     this.scannerCaching = conf.getInt("hbase.client.scanner.caching", 1);
177 
178     this.maxScannerResultSize = conf.getLong(
179       HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
180       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
181     this.maxKeyValueSize = conf.getInt("hbase.client.keyvalue.maxsize", -1);
182 
183     int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE);
184     if (maxThreads == 0) {
185       maxThreads = 1; // is there a better default?
186     }
187 
188     // Using the "direct handoff" approach, new threads will only be created
189     // if it is necessary and will grow unbounded. This could be bad but in HCM
190     // we only create as many Runnables as there are region servers. It means
191     // it also scales when new region servers are added.
192     this.pool = new ThreadPoolExecutor(1, maxThreads,
193         60, TimeUnit.SECONDS,
194         new SynchronousQueue<Runnable>(),
195         new DaemonThreadFactory());
196     ((ThreadPoolExecutor)this.pool).allowCoreThreadTimeOut(true);
197   }
198 
199   /**
200    * @return the number of region servers that are currently running
201    * @throws IOException if a remote or network exception occurs
202    */
203   public int getCurrentNrHRS() throws IOException {
204     try {
205       // We go to zk rather than to master to get count of regions to avoid
206       // HTable having a Master dependency.  See HBase-2828
207       return ZKUtil.getNumberOfChildren(this.connection.getZooKeeperWatcher(),
208           this.connection.getZooKeeperWatcher().rsZNode);
209     } catch (KeeperException ke) {
210       throw new IOException("Unexpected ZooKeeper exception", ke);
211     }
212   }
213 
214   public Configuration getConfiguration() {
215     return configuration;
216   }
217 
218   /**
219    * Tells whether or not a table is enabled or not.
220    * Warning: use {@link HBaseAdmin#isTableEnabled(byte[])} instead.
221    * @param tableName Name of table to check.
222    * @return {@code true} if table is online.
223    * @throws IOException if a remote or network exception occurs
224    */
225   public static boolean isTableEnabled(String tableName) throws IOException {
226     return isTableEnabled(Bytes.toBytes(tableName));
227   }
228 
229   /**
230    * Tells whether or not a table is enabled or not.
231    * Warning: use {@link HBaseAdmin#isTableEnabled(byte[])} instead.
232    * @param tableName Name of table to check.
233    * @return {@code true} if table is online.
234    * @throws IOException if a remote or network exception occurs
235    */
236   public static boolean isTableEnabled(byte[] tableName) throws IOException {
237     return isTableEnabled(HBaseConfiguration.create(), tableName);
238   }
239 
240   /**
241    * Tells whether or not a table is enabled or not.
242    * Warning: use {@link HBaseAdmin#isTableEnabled(byte[])} instead.
243    * @param conf The Configuration object to use.
244    * @param tableName Name of table to check.
245    * @return {@code true} if table is online.
246    * @throws IOException if a remote or network exception occurs
247    */
248   public static boolean isTableEnabled(Configuration conf, String tableName)
249   throws IOException {
250     return isTableEnabled(conf, Bytes.toBytes(tableName));
251   }
252 
253   /**
254    * Tells whether or not a table is enabled or not.
255    * @param conf The Configuration object to use.
256    * @param tableName Name of table to check.
257    * @return {@code true} if table is online.
258    * @throws IOException if a remote or network exception occurs
259    */
260   public static boolean isTableEnabled(Configuration conf, byte[] tableName)
261   throws IOException {
262     return HConnectionManager.getConnection(conf).isTableEnabled(tableName);
263   }
264 
265   /**
266    * Find region location hosting passed row using cached info
267    * @param row Row to find.
268    * @return The location of the given row.
269    * @throws IOException if a remote or network exception occurs
270    */
271   public HRegionLocation getRegionLocation(final String row)
272   throws IOException {
273     return connection.getRegionLocation(tableName, Bytes.toBytes(row), false);
274   }
275 
276   /**
277    * Finds the region on which the given row is being served.
278    * @param row Row to find.
279    * @return Location of the row.
280    * @throws IOException if a remote or network exception occurs
281    */
282   public HRegionLocation getRegionLocation(final byte [] row)
283   throws IOException {
284     return connection.getRegionLocation(tableName, row, false);
285   }
286 
287   @Override
288   public byte [] getTableName() {
289     return this.tableName;
290   }
291 
292   /**
293    * <em>INTERNAL</em> Used by unit tests and tools to do low-level
294    * manipulations.
295    * @return An HConnection instance.
296    */
297   // TODO(tsuna): Remove this.  Unit tests shouldn't require public helpers.
298   public HConnection getConnection() {
299     return this.connection;
300   }
301 
302   /**
303    * Gets the number of rows that a scanner will fetch at once.
304    * <p>
305    * The default value comes from {@code hbase.client.scanner.caching}.
306    */
307   public int getScannerCaching() {
308     return scannerCaching;
309   }
310 
311   /**
312    * Sets the number of rows that a scanner will fetch at once.
313    * <p>
314    * This will override the value specified by
315    * {@code hbase.client.scanner.caching}.
316    * Increasing this value will reduce the amount of work needed each time
317    * {@code next()} is called on a scanner, at the expense of memory use
318    * (since more rows will need to be maintained in memory by the scanners).
319    * @param scannerCaching the number of rows a scanner will fetch at once.
320    */
321   public void setScannerCaching(int scannerCaching) {
322     this.scannerCaching = scannerCaching;
323   }
324 
325   @Override
326   public HTableDescriptor getTableDescriptor() throws IOException {
327     return new UnmodifyableHTableDescriptor(
328       this.connection.getHTableDescriptor(this.tableName));
329   }
330 
331   /**
332    * Gets the starting row key for every region in the currently open table.
333    * <p>
334    * This is mainly useful for the MapReduce integration.
335    * @return Array of region starting row keys
336    * @throws IOException if a remote or network exception occurs
337    */
338   public byte [][] getStartKeys() throws IOException {
339     return getStartEndKeys().getFirst();
340   }
341 
342   /**
343    * Gets the ending row key for every region in the currently open table.
344    * <p>
345    * This is mainly useful for the MapReduce integration.
346    * @return Array of region ending row keys
347    * @throws IOException if a remote or network exception occurs
348    */
349   public byte[][] getEndKeys() throws IOException {
350     return getStartEndKeys().getSecond();
351   }
352 
353   /**
354    * Gets the starting and ending row keys for every region in the currently
355    * open table.
356    * <p>
357    * This is mainly useful for the MapReduce integration.
358    * @return Pair of arrays of region starting and ending row keys
359    * @throws IOException if a remote or network exception occurs
360    */
361   @SuppressWarnings("unchecked")
362   public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException {
363     final List<byte[]> startKeyList = new ArrayList<byte[]>();
364     final List<byte[]> endKeyList = new ArrayList<byte[]>();
365     MetaScannerVisitor visitor = new MetaScannerVisitor() {
366       public boolean processRow(Result rowResult) throws IOException {
367         byte [] bytes = rowResult.getValue(HConstants.CATALOG_FAMILY,
368           HConstants.REGIONINFO_QUALIFIER);
369         if (bytes == null) {
370           LOG.warn("Null " + HConstants.REGIONINFO_QUALIFIER + " cell in " +
371             rowResult);
372           return true;
373         }
374         HRegionInfo info = Writables.getHRegionInfo(bytes);
375         if (Bytes.equals(info.getTableDesc().getName(), getTableName())) {
376           if (!(info.isOffline() || info.isSplit())) {
377             startKeyList.add(info.getStartKey());
378             endKeyList.add(info.getEndKey());
379           }
380         }
381         return true;
382       }
383     };
384     MetaScanner.metaScan(configuration, visitor, this.tableName);
385     return new Pair(startKeyList.toArray(new byte[startKeyList.size()][]),
386                 endKeyList.toArray(new byte[endKeyList.size()][]));
387   }
388 
389   /**
390    * Gets all the regions and their address for this table.
391    * <p>
392    * This is mainly useful for the MapReduce integration.
393    * @return A map of HRegionInfo with it's server address
394    * @throws IOException if a remote or network exception occurs
395    */
396   public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
397     final Map<HRegionInfo, HServerAddress> regionMap =
398       new TreeMap<HRegionInfo, HServerAddress>();
399 
400     MetaScannerVisitor visitor = new MetaScannerVisitor() {
401       public boolean processRow(Result rowResult) throws IOException {
402         HRegionInfo info = Writables.getHRegionInfo(
403             rowResult.getValue(HConstants.CATALOG_FAMILY,
404                 HConstants.REGIONINFO_QUALIFIER));
405 
406         if (!(Bytes.equals(info.getTableDesc().getName(), getTableName()))) {
407           return false;
408         }
409 
410         HServerAddress server = new HServerAddress();
411         byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
412             HConstants.SERVER_QUALIFIER);
413         if (value != null && value.length > 0) {
414           String address = Bytes.toString(value);
415           server = new HServerAddress(address);
416         }
417 
418         if (!(info.isOffline() || info.isSplit())) {
419           regionMap.put(new UnmodifyableHRegionInfo(info), server);
420         }
421         return true;
422       }
423 
424     };
425     MetaScanner.metaScan(configuration, visitor, tableName);
426     return regionMap;
427   }
428 
429   /**
430    * Save the passed region information and the table's regions
431    * cache.
432    * <p>
433    * This is mainly useful for the MapReduce integration. You can call
434    * {@link #deserializeRegionInfo deserializeRegionInfo}
435    * to deserialize regions information from a
436    * {@link DataInput}, then call this method to load them to cache.
437    *
438    * <pre>
439    * {@code
440    * HTable t1 = new HTable("foo");
441    * FileInputStream fis = new FileInputStream("regions.dat");
442    * DataInputStream dis = new DataInputStream(fis);
443    *
444    * Map<HRegionInfo, HServerAddress> hm = t1.deserializeRegionInfo(dis);
445    * t1.prewarmRegionCache(hm);
446    * }
447    * </pre>
448    * @param regionMap This piece of regions information will be loaded
449    * to region cache.
450    */
451   public void prewarmRegionCache(Map<HRegionInfo, HServerAddress> regionMap) {
452     this.connection.prewarmRegionCache(this.getTableName(), regionMap);
453   }
454 
455   /**
456    * Serialize the regions information of this table and output
457    * to <code>out</code>.
458    * <p>
459    * This is mainly useful for the MapReduce integration. A client could
460    * perform a large scan for all the regions for the table, serialize the
461    * region info to a file. MR job can ship a copy of the meta for the table in
462    * the DistributedCache.
463    * <pre>
464    * {@code
465    * FileOutputStream fos = new FileOutputStream("regions.dat");
466    * DataOutputStream dos = new DataOutputStream(fos);
467    * table.serializeRegionInfo(dos);
468    * dos.flush();
469    * dos.close();
470    * }
471    * </pre>
472    * @param out {@link DataOutput} to serialize this object into.
473    * @throws IOException if a remote or network exception occurs
474    */
475   public void serializeRegionInfo(DataOutput out) throws IOException {
476     Map<HRegionInfo, HServerAddress> allRegions = this.getRegionsInfo();
477     // first, write number of regions
478     out.writeInt(allRegions.size());
479     for (Map.Entry<HRegionInfo, HServerAddress> es : allRegions.entrySet()) {
480       es.getKey().write(out);
481       es.getValue().write(out);
482     }
483   }
484 
485   /**
486    * Read from <code>in</code> and deserialize the regions information.
487    *
488    * <p>It behaves similarly as {@link #getRegionsInfo getRegionsInfo}, except
489    * that it loads the region map from a {@link DataInput} object.
490    *
491    * <p>It is supposed to be followed immediately by  {@link
492    * #prewarmRegionCache prewarmRegionCache}.
493    *
494    * <p>
495    * Please refer to {@link #prewarmRegionCache prewarmRegionCache} for usage.
496    *
497    * @param in {@link DataInput} object.
498    * @return A map of HRegionInfo with its server address.
499    * @throws IOException if an I/O exception occurs.
500    */
501   public Map<HRegionInfo, HServerAddress> deserializeRegionInfo(DataInput in)
502   throws IOException {
503     final Map<HRegionInfo, HServerAddress> allRegions =
504       new TreeMap<HRegionInfo, HServerAddress>();
505 
506     // the first integer is expected to be the size of records
507     int regionsCount = in.readInt();
508     for (int i = 0; i < regionsCount; ++i) {
509       HRegionInfo hri = new HRegionInfo();
510       hri.readFields(in);
511       HServerAddress hsa = new HServerAddress();
512       hsa.readFields(in);
513       allRegions.put(hri, hsa);
514     }
515     return allRegions;
516   }
517 
518    @Override
519    public Result getRowOrBefore(final byte[] row, final byte[] family)
520    throws IOException {
521      return connection.getRegionServerWithRetries(
522          new ServerCallable<Result>(connection, tableName, row) {
523        public Result call() throws IOException {
524          return server.getClosestRowBefore(location.getRegionInfo().getRegionName(),
525            row, family);
526        }
527      });
528    }
529 
530   @Override
531   public ResultScanner getScanner(final Scan scan) throws IOException {
532     ClientScanner s = new ClientScanner(scan);
533     s.initialize();
534     return s;
535   }
536 
537   @Override
538   public ResultScanner getScanner(byte [] family) throws IOException {
539     Scan scan = new Scan();
540     scan.addFamily(family);
541     return getScanner(scan);
542   }
543 
544   @Override
545   public ResultScanner getScanner(byte [] family, byte [] qualifier)
546   throws IOException {
547     Scan scan = new Scan();
548     scan.addColumn(family, qualifier);
549     return getScanner(scan);
550   }
551 
552   public Result get(final Get get) throws IOException {
553     return connection.getRegionServerWithRetries(
554         new ServerCallable<Result>(connection, tableName, get.getRow()) {
555           public Result call() throws IOException {
556             return server.get(location.getRegionInfo().getRegionName(), get);
557           }
558         }
559     );
560   }
561 
562    public Result[] get(List<Get> gets) throws IOException {
563      try {
564        Object [] r1 = batch((List)gets);
565 
566        // translate.
567        Result [] results = new Result[r1.length];
568        int i=0;
569        for (Object o : r1) {
570          // batch ensures if there is a failure we get an exception instead
571          results[i++] = (Result) o;
572        }
573 
574        return results;
575      } catch (InterruptedException e) {
576        throw new IOException(e);
577      }
578    }
579 
580   /**
581    * Method that does a batch call on Deletes, Gets and Puts.  The ordering of
582    * execution of the actions is not defined. Meaning if you do a Put and a
583    * Get in the same {@link #batch} call, you will not necessarily be
584    * guaranteed that the Get returns what the Put had put.
585    *
586    * @param actions list of Get, Put, Delete objects
587    * @param results Empty Result[], same size as actions. Provides access to
588    * partial results, in case an exception is thrown. If there are any failures,
589    * there will be a null or Throwable will be in the results array, AND an
590    * exception will be thrown.
591    * @throws IOException
592    */
593   @Override
594   public synchronized void batch(final List<Row> actions, final Object[] results)
595       throws InterruptedException, IOException {
596     connection.processBatch(actions, tableName, pool, results);
597   }
598 
599   /**
600    * Method that does a batch call on Deletes, Gets and Puts.
601    *
602    * @param actions list of Get, Put, Delete objects
603    * @return the results from the actions. A null in the return array means that
604    * the call for that action failed, even after retries
605    * @throws IOException
606    */
607   @Override
608   public synchronized Object[] batch(final List<Row> actions) throws InterruptedException, IOException {
609     Object[] results = new Object[actions.size()];
610     connection.processBatch(actions, tableName, pool, results);
611     return results;
612   }
613 
614   /**
615    * Deletes the specified cells/row.
616    *
617    * @param delete The object that specifies what to delete.
618    * @throws IOException if a remote or network exception occurs.
619    * @since 0.20.0
620    */
621   @Override
622   public void delete(final Delete delete)
623   throws IOException {
624     connection.getRegionServerWithRetries(
625         new ServerCallable<Boolean>(connection, tableName, delete.getRow()) {
626           public Boolean call() throws IOException {
627             server.delete(location.getRegionInfo().getRegionName(), delete);
628             return null; // FindBugs NP_BOOLEAN_RETURN_NULL
629           }
630         }
631     );
632   }
633 
634   /**
635    * Deletes the specified cells/rows in bulk.
636    * @param deletes List of things to delete. As a side effect, it will be modified:
637    * successful {@link Delete}s are removed. The ordering of the list will not change.
638    * @throws IOException if a remote or network exception occurs. In that case
639    * the {@code deletes} argument will contain the {@link Delete} instances
640    * that have not be successfully applied.
641    * @since 0.20.1
642    * @see #batch(java.util.List, Object[])
643    */
644   @Override
645   public void delete(final List<Delete> deletes)
646   throws IOException {
647     Object[] results = new Object[deletes.size()];
648     try {
649       connection.processBatch((List) deletes, tableName, pool, results);
650     } catch (InterruptedException e) {
651       throw new IOException(e);
652     } finally {
653       // mutate list so that it is empty for complete success, or contains only failed records
654       // results are returned in the same order as the requests in list
655       // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
656       for (int i = results.length - 1; i>=0; i--) {
657         // if result is not null, it succeeded
658         if (results[i] instanceof Result) {
659           deletes.remove(i);
660         }
661       }
662     }
663   }
664 
665   @Override
666   public void put(final Put put) throws IOException {
667     doPut(Arrays.asList(put));
668   }
669 
670   @Override
671   public void put(final List<Put> puts) throws IOException {
672     doPut(puts);
673   }
674 
675   private void doPut(final List<Put> puts) throws IOException {
676     for (Put put : puts) {
677       validatePut(put);
678       writeBuffer.add(put);
679       currentWriteBufferSize += put.heapSize();
680     }
681     if (autoFlush || currentWriteBufferSize > writeBufferSize) {
682       flushCommits();
683     }
684   }
685 
686   @Override
687   public Result increment(final Increment increment) throws IOException {
688     if (!increment.hasFamilies()) {
689       throw new IOException(
690           "Invalid arguments to increment, no columns specified");
691     }
692     return connection.getRegionServerWithRetries(
693         new ServerCallable<Result>(connection, tableName, increment.getRow()) {
694           public Result call() throws IOException {
695             return server.increment(
696                 location.getRegionInfo().getRegionName(), increment);
697           }
698         }
699     );
700   }
701 
702   @Override
703   public long incrementColumnValue(final byte [] row, final byte [] family,
704       final byte [] qualifier, final long amount)
705   throws IOException {
706     return incrementColumnValue(row, family, qualifier, amount, true);
707   }
708 
709   @Override
710   public long incrementColumnValue(final byte [] row, final byte [] family,
711       final byte [] qualifier, final long amount, final boolean writeToWAL)
712   throws IOException {
713     NullPointerException npe = null;
714     if (row == null) {
715       npe = new NullPointerException("row is null");
716     } else if (family == null) {
717       npe = new NullPointerException("column is null");
718     }
719     if (npe != null) {
720       throw new IOException(
721           "Invalid arguments to incrementColumnValue", npe);
722     }
723     return connection.getRegionServerWithRetries(
724         new ServerCallable<Long>(connection, tableName, row) {
725           public Long call() throws IOException {
726             return server.incrementColumnValue(
727                 location.getRegionInfo().getRegionName(), row, family,
728                 qualifier, amount, writeToWAL);
729           }
730         }
731     );
732   }
733 
734   /**
735    * Atomically checks if a row/family/qualifier value match the expectedValue.
736    * If it does, it adds the put.  If value == null, checks for non-existence
737    * of the value.
738    *
739    * @param row to check
740    * @param family column family
741    * @param qualifier column qualifier
742    * @param value the expected value
743    * @param put put to execute if value matches.
744    * @throws IOException
745    * @return true if the new put was execute, false otherwise
746    */
747   @Override
748   public boolean checkAndPut(final byte [] row,
749       final byte [] family, final byte [] qualifier, final byte [] value,
750       final Put put)
751   throws IOException {
752     return connection.getRegionServerWithRetries(
753         new ServerCallable<Boolean>(connection, tableName, row) {
754           public Boolean call() throws IOException {
755             return server.checkAndPut(location.getRegionInfo().getRegionName(),
756                 row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE;
757           }
758         }
759     );
760   }
761 
762   /**
763    * Atomically checks if a row/family/qualifier value match the expectedValue.
764    * If it does, it adds the delete.  If value == null, checks for non-existence
765    * of the value.
766    *
767    * @param row to check
768    * @param family column family
769    * @param qualifier column qualifier
770    * @param value the expected value
771    * @param delete delete to execute if value matches.
772    * @throws IOException
773    * @return true if the new delete was executed, false otherwise
774    */
775   @Override
776   public boolean checkAndDelete(final byte [] row,
777       final byte [] family, final byte [] qualifier, final byte [] value,
778       final Delete delete)
779   throws IOException {
780     return connection.getRegionServerWithRetries(
781         new ServerCallable<Boolean>(connection, tableName, row) {
782           public Boolean call() throws IOException {
783             return server.checkAndDelete(
784                 location.getRegionInfo().getRegionName(),
785                 row, family, qualifier, value, delete)
786             ? Boolean.TRUE : Boolean.FALSE;
787           }
788         }
789     );
790   }
791 
792   /**
793    * Test for the existence of columns in the table, as specified in the Get.<p>
794    *
795    * This will return true if the Get matches one or more keys, false if not.<p>
796    *
797    * This is a server-side call so it prevents any data from being transfered
798    * to the client.
799    * @param get param to check for
800    * @return true if the specified Get matches one or more keys, false if not
801    * @throws IOException
802    */
803   @Override
804   public boolean exists(final Get get) throws IOException {
805     return connection.getRegionServerWithRetries(
806         new ServerCallable<Boolean>(connection, tableName, get.getRow()) {
807           public Boolean call() throws IOException {
808             return server.
809                 exists(location.getRegionInfo().getRegionName(), get);
810           }
811         }
812     );
813   }
814 
815   /**
816    * Executes all the buffered {@link Put} operations.
817    * <p>
818    * This method gets called once automatically for every {@link Put} or batch
819    * of {@link Put}s (when {@link #batch(List)} is used) when
820    * {@link #isAutoFlush()} is {@code true}.
821    * @throws IOException if a remote or network exception occurs.
822    */
823   @Override
824   public void flushCommits() throws IOException {
825     try {
826       connection.processBatchOfPuts(writeBuffer, tableName, pool);
827     } finally {
828       // the write buffer was adjusted by processBatchOfPuts
829       currentWriteBufferSize = 0;
830       for (Put aPut : writeBuffer) {
831         currentWriteBufferSize += aPut.heapSize();
832       }
833     }
834   }
835 
836   @Override
837   public void close() throws IOException {
838     flushCommits();
839     this.pool.shutdown();
840   }
841 
842   // validate for well-formedness
843   private void validatePut(final Put put) throws IllegalArgumentException{
844     if (put.isEmpty()) {
845       throw new IllegalArgumentException("No columns to insert");
846     }
847     if (maxKeyValueSize > 0) {
848       for (List<KeyValue> list : put.getFamilyMap().values()) {
849         for (KeyValue kv : list) {
850           if (kv.getLength() > maxKeyValueSize) {
851             throw new IllegalArgumentException("KeyValue size too large");
852           }
853         }
854       }
855     }
856   }
857 
858   @Override
859   public RowLock lockRow(final byte [] row)
860   throws IOException {
861     return connection.getRegionServerWithRetries(
862       new ServerCallable<RowLock>(connection, tableName, row) {
863         public RowLock call() throws IOException {
864           long lockId =
865               server.lockRow(location.getRegionInfo().getRegionName(), row);
866           return new RowLock(row,lockId);
867         }
868       }
869     );
870   }
871 
872   @Override
873   public void unlockRow(final RowLock rl)
874   throws IOException {
875     connection.getRegionServerWithRetries(
876       new ServerCallable<Boolean>(connection, tableName, rl.getRow()) {
877         public Boolean call() throws IOException {
878           server.unlockRow(location.getRegionInfo().getRegionName(),
879               rl.getLockId());
880           return null; // FindBugs NP_BOOLEAN_RETURN_NULL
881         }
882       }
883     );
884   }
885 
886   /**
887    * Explicitly clears the region cache to fetch the latest value from META.
888    * This is a power user function: avoid unless you know the ramifications.
889    */
890   public void clearRegionCache() {
891     this.connection.clearRegionCache();
892   }
893 
894   @Override
895   public boolean isAutoFlush() {
896     return autoFlush;
897   }
898 
899   /**
900    * Turns 'auto-flush' on or off.
901    * <p>
902    * When enabled (default), {@link Put} operations don't get buffered/delayed
903    * and are immediately executed.  This is slower but safer.
904    * <p>
905    * Turning this off means that multiple {@link Put}s will be accepted before
906    * any RPC is actually sent to do the write operations.  If the application
907    * dies before pending writes get flushed to HBase, data will be lost.
908    * Other side effects may include the fact that the application thinks a
909    * {@link Put} was executed successfully whereas it was in fact only
910    * buffered and the operation may fail when attempting to flush all pending
911    * writes.  In that case though, the code will retry the failed {@link Put}
912    * upon its next attempt to flush the buffer.
913    *
914    * @param autoFlush Whether or not to enable 'auto-flush'.
915    * @see #flushCommits
916    */
917   public void setAutoFlush(boolean autoFlush) {
918     this.autoFlush = autoFlush;
919   }
920 
921   /**
922    * Returns the maximum size in bytes of the write buffer for this HTable.
923    * <p>
924    * The default value comes from the configuration parameter
925    * {@code hbase.client.write.buffer}.
926    * @return The size of the write buffer in bytes.
927    */
928   public long getWriteBufferSize() {
929     return writeBufferSize;
930   }
931 
932   /**
933    * Sets the size of the buffer in bytes.
934    * <p>
935    * If the new size is less than the current amount of data in the
936    * write buffer, the buffer gets flushed.
937    * @param writeBufferSize The new write buffer size, in bytes.
938    * @throws IOException if a remote or network exception occurs.
939    */
940   public void setWriteBufferSize(long writeBufferSize) throws IOException {
941     this.writeBufferSize = writeBufferSize;
942     if(currentWriteBufferSize > writeBufferSize) {
943       flushCommits();
944     }
945   }
946 
947   /**
948    * Returns the write buffer.
949    * @return The current write buffer.
950    */
951   public ArrayList<Put> getWriteBuffer() {
952     return writeBuffer;
953   }
954 
955   /**
956    * Implements the scanner interface for the HBase client.
957    * If there are multiple regions in a table, this scanner will iterate
958    * through them all.
959    */
960   protected class ClientScanner implements ResultScanner {
961     private final Log CLIENT_LOG = LogFactory.getLog(this.getClass());
962     // HEADSUP: The scan internal start row can change as we move through table.
963     private Scan scan;
964     private boolean closed = false;
965     // Current region scanner is against.  Gets cleared if current region goes
966     // wonky: e.g. if it splits on us.
967     private HRegionInfo currentRegion = null;
968     private ScannerCallable callable = null;
969     private final LinkedList<Result> cache = new LinkedList<Result>();
970     private final int caching;
971     private long lastNext;
972     // Keep lastResult returned successfully in case we have to reset scanner.
973     private Result lastResult = null;
974 
975     protected ClientScanner(final Scan scan) {
976       if (CLIENT_LOG.isDebugEnabled()) {
977         CLIENT_LOG.debug("Creating scanner over "
978             + Bytes.toString(getTableName())
979             + " starting at key '" + Bytes.toStringBinary(scan.getStartRow()) + "'");
980       }
981       this.scan = scan;
982       this.lastNext = System.currentTimeMillis();
983 
984       // Use the caching from the Scan.  If not set, use the default cache setting for this table.
985       if (this.scan.getCaching() > 0) {
986         this.caching = this.scan.getCaching();
987       } else {
988         this.caching = HTable.this.scannerCaching;
989       }
990 
991       // Removed filter validation.  We have a new format now, only one of all
992       // the current filters has a validate() method.  We can add it back,
993       // need to decide on what we're going to do re: filter redesign.
994       // Need, at the least, to break up family from qualifier as separate
995       // checks, I think it's important server-side filters are optimal in that
996       // respect.
997     }
998 
999     public void initialize() throws IOException {
1000       nextScanner(this.caching, false);
1001     }
1002 
1003     protected Scan getScan() {
1004       return scan;
1005     }
1006 
1007     protected long getTimestamp() {
1008       return lastNext;
1009     }
1010 
1011     // returns true if the passed region endKey
1012     private boolean checkScanStopRow(final byte [] endKey) {
1013       if (this.scan.getStopRow().length > 0) {
1014         // there is a stop row, check to see if we are past it.
1015         byte [] stopRow = scan.getStopRow();
1016         int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
1017           endKey, 0, endKey.length);
1018         if (cmp <= 0) {
1019           // stopRow <= endKey (endKey is equals to or larger than stopRow)
1020           // This is a stop.
1021           return true;
1022         }
1023       }
1024       return false; //unlikely.
1025     }
1026 
1027     /*
1028      * Gets a scanner for the next region.  If this.currentRegion != null, then
1029      * we will move to the endrow of this.currentRegion.  Else we will get
1030      * scanner at the scan.getStartRow().  We will go no further, just tidy
1031      * up outstanding scanners, if <code>currentRegion != null</code> and
1032      * <code>done</code> is true.
1033      * @param nbRows
1034      * @param done Server-side says we're done scanning.
1035      */
1036     private boolean nextScanner(int nbRows, final boolean done)
1037     throws IOException {
1038       // Close the previous scanner if it's open
1039       if (this.callable != null) {
1040         this.callable.setClose();
1041         getConnection().getRegionServerWithRetries(callable);
1042         this.callable = null;
1043       }
1044 
1045       // Where to start the next scanner
1046       byte [] localStartKey;
1047 
1048       // if we're at end of table, close and return false to stop iterating
1049       if (this.currentRegion != null) {
1050         byte [] endKey = this.currentRegion.getEndKey();
1051         if (endKey == null ||
1052             Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
1053             checkScanStopRow(endKey) ||
1054             done) {
1055           close();
1056           if (CLIENT_LOG.isDebugEnabled()) {
1057             CLIENT_LOG.debug("Finished with scanning at " + this.currentRegion);
1058           }
1059           return false;
1060         }
1061         localStartKey = endKey;
1062         if (CLIENT_LOG.isDebugEnabled()) {
1063           CLIENT_LOG.debug("Finished with region " + this.currentRegion);
1064         }
1065       } else {
1066         localStartKey = this.scan.getStartRow();
1067       }
1068 
1069       if (CLIENT_LOG.isDebugEnabled()) {
1070         CLIENT_LOG.debug("Advancing internal scanner to startKey at '" +
1071           Bytes.toStringBinary(localStartKey) + "'");
1072       }
1073       try {
1074         callable = getScannerCallable(localStartKey, nbRows);
1075         // Open a scanner on the region server starting at the
1076         // beginning of the region
1077         getConnection().getRegionServerWithRetries(callable);
1078         this.currentRegion = callable.getHRegionInfo();
1079       } catch (IOException e) {
1080         close();
1081         throw e;
1082       }
1083       return true;
1084     }
1085 
1086     protected ScannerCallable getScannerCallable(byte [] localStartKey,
1087         int nbRows) {
1088       scan.setStartRow(localStartKey);
1089       ScannerCallable s = new ScannerCallable(getConnection(),
1090         getTableName(), scan);
1091       s.setCaching(nbRows);
1092       return s;
1093     }
1094 
1095     public Result next() throws IOException {
1096       // If the scanner is closed but there is some rows left in the cache,
1097       // it will first empty it before returning null
1098       if (cache.size() == 0 && this.closed) {
1099         return null;
1100       }
1101       if (cache.size() == 0) {
1102         Result [] values = null;
1103         long remainingResultSize = maxScannerResultSize;
1104         int countdown = this.caching;
1105         // We need to reset it if it's a new callable that was created
1106         // with a countdown in nextScanner
1107         callable.setCaching(this.caching);
1108         // This flag is set when we want to skip the result returned.  We do
1109         // this when we reset scanner because it split under us.
1110         boolean skipFirst = false;
1111         do {
1112           try {
1113             if (skipFirst) {
1114               // Skip only the first row (which was the last row of the last
1115               // already-processed batch).
1116               callable.setCaching(1);
1117               values = getConnection().getRegionServerWithRetries(callable);
1118               callable.setCaching(this.caching);
1119               skipFirst = false;
1120             }
1121             // Server returns a null values if scanning is to stop.  Else,
1122             // returns an empty array if scanning is to go on and we've just
1123             // exhausted current region.
1124             values = getConnection().getRegionServerWithRetries(callable);
1125           } catch (DoNotRetryIOException e) {
1126             if (e instanceof UnknownScannerException) {
1127               long timeout = lastNext + scannerTimeout;
1128               // If we are over the timeout, throw this exception to the client
1129               // Else, it's because the region moved and we used the old id
1130               // against the new region server; reset the scanner.
1131               if (timeout < System.currentTimeMillis()) {
1132                 long elapsed = System.currentTimeMillis() - lastNext;
1133                 ScannerTimeoutException ex = new ScannerTimeoutException(
1134                     elapsed + "ms passed since the last invocation, " +
1135                         "timeout is currently set to " + scannerTimeout);
1136                 ex.initCause(e);
1137                 throw ex;
1138               }
1139             } else {
1140               Throwable cause = e.getCause();
1141               if (cause == null || !(cause instanceof NotServingRegionException)) {
1142                 throw e;
1143               }
1144             }
1145             // Else, its signal from depths of ScannerCallable that we got an
1146             // NSRE on a next and that we need to reset the scanner.
1147             if (this.lastResult != null) {
1148               this.scan.setStartRow(this.lastResult.getRow());
1149               // Skip first row returned.  We already let it out on previous
1150               // invocation.
1151               skipFirst = true;
1152             }
1153             // Clear region
1154             this.currentRegion = null;
1155             continue;
1156           }
1157           lastNext = System.currentTimeMillis();
1158           if (values != null && values.length > 0) {
1159             for (Result rs : values) {
1160               cache.add(rs);
1161               for (KeyValue kv : rs.raw()) {
1162                   remainingResultSize -= kv.heapSize();
1163               }
1164               countdown--;
1165               this.lastResult = rs;
1166             }
1167           }
1168           // Values == null means server-side filter has determined we must STOP
1169         } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
1170       }
1171 
1172       if (cache.size() > 0) {
1173         return cache.poll();
1174       }
1175       return null;
1176     }
1177 
1178     /**
1179      * Get <param>nbRows</param> rows.
1180      * How many RPCs are made is determined by the {@link Scan#setCaching(int)}
1181      * setting (or hbase.client.scanner.caching in hbase-site.xml).
1182      * @param nbRows number of rows to return
1183      * @return Between zero and <param>nbRows</param> RowResults.  Scan is done
1184      * if returned array is of zero-length (We never return null).
1185      * @throws IOException
1186      */
1187     public Result [] next(int nbRows) throws IOException {
1188       // Collect values to be returned here
1189       ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
1190       for(int i = 0; i < nbRows; i++) {
1191         Result next = next();
1192         if (next != null) {
1193           resultSets.add(next);
1194         } else {
1195           break;
1196         }
1197       }
1198       return resultSets.toArray(new Result[resultSets.size()]);
1199     }
1200 
1201     public void close() {
1202       if (callable != null) {
1203         callable.setClose();
1204         try {
1205           getConnection().getRegionServerWithRetries(callable);
1206         } catch (IOException e) {
1207           // We used to catch this error, interpret, and rethrow. However, we
1208           // have since decided that it's not nice for a scanner's close to
1209           // throw exceptions. Chances are it was just an UnknownScanner
1210           // exception due to lease time out.
1211         }
1212         callable = null;
1213       }
1214       closed = true;
1215     }
1216 
1217     public Iterator<Result> iterator() {
1218       return new Iterator<Result>() {
1219         // The next RowResult, possibly pre-read
1220         Result next = null;
1221 
1222         // return true if there is another item pending, false if there isn't.
1223         // this method is where the actual advancing takes place, but you need
1224         // to call next() to consume it. hasNext() will only advance if there
1225         // isn't a pending next().
1226         public boolean hasNext() {
1227           if (next == null) {
1228             try {
1229               next = ClientScanner.this.next();
1230               return next != null;
1231             } catch (IOException e) {
1232               throw new RuntimeException(e);
1233             }
1234           }
1235           return true;
1236         }
1237 
1238         // get the pending next item and advance the iterator. returns null if
1239         // there is no next item.
1240         public Result next() {
1241           // since hasNext() does the real advancing, we call this to determine
1242           // if there is a next before proceeding.
1243           if (!hasNext()) {
1244             return null;
1245           }
1246 
1247           // if we get to here, then hasNext() has given us an item to return.
1248           // we want to return the item and then null out the next pointer, so
1249           // we use a temporary variable.
1250           Result temp = next;
1251           next = null;
1252           return temp;
1253         }
1254 
1255         public void remove() {
1256           throw new UnsupportedOperationException();
1257         }
1258       };
1259     }
1260   }
1261 
1262   /**
1263    * The pool is used for mutli requests for this HTable
1264    * @return the pool used for mutli
1265    */
1266   ExecutorService getPool() {
1267     return this.pool;
1268   }
1269 
1270   static class DaemonThreadFactory implements ThreadFactory {
1271     static final AtomicInteger poolNumber = new AtomicInteger(1);
1272         final ThreadGroup group;
1273         final AtomicInteger threadNumber = new AtomicInteger(1);
1274         final String namePrefix;
1275 
1276         DaemonThreadFactory() {
1277             SecurityManager s = System.getSecurityManager();
1278             group = (s != null)? s.getThreadGroup() :
1279                                  Thread.currentThread().getThreadGroup();
1280             namePrefix = "pool-" +
1281                           poolNumber.getAndIncrement() +
1282                          "-thread-";
1283         }
1284 
1285         public Thread newThread(Runnable r) {
1286             Thread t = new Thread(group, r,
1287                                   namePrefix + threadNumber.getAndIncrement(),
1288                                   0);
1289             if (!t.isDaemon()) {
1290               t.setDaemon(true);
1291             }
1292             if (t.getPriority() != Thread.NORM_PRIORITY) {
1293               t.setPriority(Thread.NORM_PRIORITY);
1294             }
1295             return t;
1296         }
1297   }
1298 
1299   /**
1300    * Enable or disable region cache prefetch for the table. It will be
1301    * applied for the given table's all HTable instances who share the same
1302    * connection. By default, the cache prefetch is enabled.
1303    * @param tableName name of table to configure.
1304    * @param enable Set to true to enable region cache prefetch. Or set to
1305    * false to disable it.
1306    * @throws ZooKeeperConnectionException
1307    */
1308   public static void setRegionCachePrefetch(final byte[] tableName,
1309       boolean enable) throws ZooKeeperConnectionException {
1310     HConnectionManager.getConnection(HBaseConfiguration.create()).
1311     setRegionCachePrefetch(tableName, enable);
1312   }
1313 
1314   /**
1315    * Enable or disable region cache prefetch for the table. It will be
1316    * applied for the given table's all HTable instances who share the same
1317    * connection. By default, the cache prefetch is enabled.
1318    * @param conf The Configuration object to use.
1319    * @param tableName name of table to configure.
1320    * @param enable Set to true to enable region cache prefetch. Or set to
1321    * false to disable it.
1322    * @throws ZooKeeperConnectionException
1323    */
1324   public static void setRegionCachePrefetch(final Configuration conf,
1325       final byte[] tableName, boolean enable) throws ZooKeeperConnectionException {
1326     HConnectionManager.getConnection(conf).setRegionCachePrefetch(
1327         tableName, enable);
1328   }
1329 
1330   /**
1331    * Check whether region cache prefetch is enabled or not for the table.
1332    * @param conf The Configuration object to use.
1333    * @param tableName name of table to check
1334    * @return true if table's region cache prefecth is enabled. Otherwise
1335    * it is disabled.
1336    * @throws ZooKeeperConnectionException
1337    */
1338   public static boolean getRegionCachePrefetch(final Configuration conf,
1339       final byte[] tableName) throws ZooKeeperConnectionException {
1340     return HConnectionManager.getConnection(conf).getRegionCachePrefetch(
1341         tableName);
1342   }
1343 
1344   /**
1345    * Check whether region cache prefetch is enabled or not for the table.
1346    * @param tableName name of table to check
1347    * @return true if table's region cache prefecth is enabled. Otherwise
1348    * it is disabled.
1349    * @throws ZooKeeperConnectionException
1350    */
1351   public static boolean getRegionCachePrefetch(final byte[] tableName) throws ZooKeeperConnectionException {
1352     return HConnectionManager.getConnection(HBaseConfiguration.create()).
1353     getRegionCachePrefetch(tableName);
1354   }
1355 }