View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.Iterator;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.TreeMap;
32  import java.util.concurrent.ExecutorService;
33  import java.util.concurrent.LinkedBlockingQueue;
34  import java.util.concurrent.ThreadFactory;
35  import java.util.concurrent.ThreadPoolExecutor;
36  import java.util.concurrent.TimeUnit;
37  import java.util.concurrent.atomic.AtomicInteger;
38  
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.DoNotRetryIOException;
43  import org.apache.hadoop.hbase.HBaseConfiguration;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HRegionLocation;
47  import org.apache.hadoop.hbase.HServerAddress;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.KeyValue;
50  import org.apache.hadoop.hbase.NotServingRegionException;
51  import org.apache.hadoop.hbase.UnknownScannerException;
52  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
53  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
54  import org.apache.hadoop.hbase.util.Bytes;
55  import org.apache.hadoop.hbase.util.Pair;
56  import org.apache.hadoop.hbase.util.Writables;
57  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
58  import org.apache.zookeeper.KeeperException;
59  
60  /**
61   * Used to communicate with a single HBase table.
62   *
63   * This class is not thread safe for updates; the underlying write buffer can
64   * be corrupted if multiple threads contend over a single HTable instance.
65   *
66   * <p>Instances of HTable passed the same {@link Configuration} instance will
67   * share connections to servers out on the cluster and to the zookeeper ensemble
68   * as well as caches of  region locations.  This is usually a *good* thing.
69   * This happens because they will all share the same underlying
70   * {@link HConnection} instance.  See {@link HConnectionManager} for more on
71   * how this mechanism works.
72   *
73   * <p>{@link HConnection} will read most of the
74   * configuration it needs from the passed {@link Configuration} on initial
75   * construction.  Thereafter, for settings such as
76   * <code>hbase.client.pause</code>, <code>hbase.client.retries.number</code>,
77   * and <code>hbase.client.rpc.maxattempts</code> updating their values in the
78   * passed {@link Configuration} subsequent to {@link HConnection} construction
79   * will go unnoticed.  To run with changed values, make a new
80   * {@link HTable} passing a new {@link Configuration} instance that has the
81   * new configuration.
82   *
83   * @see HBaseAdmin for create, drop, list, enable and disable of tables.
84   * @see HConnection
85   * @see HConnectionManager
86   */
87  public class HTable implements HTableInterface {
88    private static final Log LOG = LogFactory.getLog(HTable.class);
89    private final HConnection connection;
90    private final byte [] tableName;
91    protected final int scannerTimeout;
92    private volatile Configuration configuration;
93    private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
94    private long writeBufferSize;
95    private boolean autoFlush;
96    private long currentWriteBufferSize;
97    protected int scannerCaching;
98    private int maxKeyValueSize;
99    private ExecutorService pool;  // For Multi
100   private long maxScannerResultSize;
101 
102   /**
103    * Creates an object to access a HBase table.
104    * Internally it creates a new instance of {@link Configuration} and a new
105    * client to zookeeper as well as other resources.  It also comes up with
106    * a fresh view of the cluster and must do discovery from scratch of region
107    * locations; i.e. it will not make use of already-cached region locations if
108    * available. Use only when being quick and dirty.
109    * @throws IOException if a remote or network exception occurs
110    * @see #HTable(Configuration, String)
111    */
112   public HTable(final String tableName)
113   throws IOException {
114     this(HBaseConfiguration.create(), Bytes.toBytes(tableName));
115   }
116 
117   /**
118    * Creates an object to access a HBase table.
119    * Internally it creates a new instance of {@link Configuration} and a new
120    * client to zookeeper as well as other resources.  It also comes up with
121    * a fresh view of the cluster and must do discovery from scratch of region
122    * locations; i.e. it will not make use of already-cached region locations if
123    * available. Use only when being quick and dirty.
124    * @param tableName Name of the table.
125    * @throws IOException if a remote or network exception occurs
126    * @see #HTable(Configuration, String)
127    */
128   public HTable(final byte [] tableName)
129   throws IOException {
130     this(HBaseConfiguration.create(), tableName);
131   }
132 
133   /**
134    * Creates an object to access a HBase table.
135    * Shares zookeeper connection and other resources with other HTable instances
136    * created with the same <code>conf</code> instance.  Uses already-populated
137    * region cache if one is available, populated by any other HTable instances
138    * sharing this <code>conf</code> instance.  Recommended.
139    * @param conf Configuration object to use.
140    * @param tableName Name of the table.
141    * @throws IOException if a remote or network exception occurs
142    */
143   public HTable(Configuration conf, final String tableName)
144   throws IOException {
145     this(conf, Bytes.toBytes(tableName));
146   }
147 
148 
149   /**
150    * Creates an object to access a HBase table.
151    * Shares zookeeper connection and other resources with other HTable instances
152    * created with the same <code>conf</code> instance.  Uses already-populated
153    * region cache if one is available, populated by any other HTable instances
154    * sharing this <code>conf</code> instance.  Recommended.
155    * @param conf Configuration object to use.
156    * @param tableName Name of the table.
157    * @throws IOException if a remote or network exception occurs
158    */
159   public HTable(Configuration conf, final byte [] tableName)
160   throws IOException {
161     this.tableName = tableName;
162     if (conf == null) {
163       this.scannerTimeout = 0;
164       this.connection = null;
165       return;
166     }
167     this.connection = HConnectionManager.getConnection(conf);
168     this.scannerTimeout =
169       (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
170     this.configuration = conf;
171     this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
172     this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152);
173     this.autoFlush = true;
174     this.currentWriteBufferSize = 0;
175     this.scannerCaching = conf.getInt("hbase.client.scanner.caching", 1);
176 
177     this.maxScannerResultSize = conf.getLong(
178       HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
179       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
180     this.maxKeyValueSize = conf.getInt("hbase.client.keyvalue.maxsize", -1);
181 
182     int nrThreads = conf.getInt("hbase.htable.threads.max", getCurrentNrHRS());
183     if (nrThreads == 0) {
184       nrThreads = 1; // is there a better default?
185     }
186 
187     // Unfortunately Executors.newCachedThreadPool does not allow us to
188     // set the maximum size of the pool, so we have to do it ourselves.
189     // Must also set set corethreadpool size as with a LinkedBlockingQueue,
190     // a new thread will not be started until the queue is full
191     this.pool = new ThreadPoolExecutor(nrThreads, nrThreads,
192         60, TimeUnit.SECONDS,
193         new LinkedBlockingQueue<Runnable>(),
194         new DaemonThreadFactory());
195     ((ThreadPoolExecutor)this.pool).allowCoreThreadTimeOut(true);
196   }
197 
198   public Configuration getConfiguration() {
199     return configuration;
200   }
201 
202   /**
203    * @return the number of region servers that are currently running
204    * @throws IOException if a remote or network exception occurs
205    */
206   public int getCurrentNrHRS() throws IOException {
207     try {
208       // We go to zk rather than to master to get count of regions to avoid
209       // HTable having a Master dependency.  See HBase-2828
210       return ZKUtil.getNumberOfChildren(this.connection.getZooKeeperWatcher(),
211           this.connection.getZooKeeperWatcher().rsZNode);
212     } catch (KeeperException ke) {
213       throw new IOException("Unexpected ZooKeeper exception", ke);
214     }
215   }
216 
217   /**
218    * Tells whether or not a table is enabled or not.
219    * @param tableName Name of table to check.
220    * @return {@code true} if table is online.
221    * @throws IOException if a remote or network exception occurs
222    */
223   public static boolean isTableEnabled(String tableName) throws IOException {
224     return isTableEnabled(Bytes.toBytes(tableName));
225   }
226 
227   /**
228    * Tells whether or not a table is enabled or not.
229    * @param tableName Name of table to check.
230    * @return {@code true} if table is online.
231    * @throws IOException if a remote or network exception occurs
232    */
233   public static boolean isTableEnabled(byte[] tableName) throws IOException {
234     return isTableEnabled(HBaseConfiguration.create(), tableName);
235   }
236 
237   /**
238    * Tells whether or not a table is enabled or not.
239    * @param conf The Configuration object to use.
240    * @param tableName Name of table to check.
241    * @return {@code true} if table is online.
242    * @throws IOException if a remote or network exception occurs
243    */
244   public static boolean isTableEnabled(Configuration conf, String tableName)
245   throws IOException {
246     return isTableEnabled(conf, Bytes.toBytes(tableName));
247   }
248 
249   /**
250    * Tells whether or not a table is enabled or not.
251    * @param conf The Configuration object to use.
252    * @param tableName Name of table to check.
253    * @return {@code true} if table is online.
254    * @throws IOException if a remote or network exception occurs
255    */
256   public static boolean isTableEnabled(Configuration conf, byte[] tableName)
257   throws IOException {
258     return HConnectionManager.getConnection(conf).isTableEnabled(tableName);
259   }
260 
261   /**
262    * Find region location hosting passed row using cached info
263    * @param row Row to find.
264    * @return The location of the given row.
265    * @throws IOException if a remote or network exception occurs
266    */
267   public HRegionLocation getRegionLocation(final String row)
268   throws IOException {
269     return connection.getRegionLocation(tableName, Bytes.toBytes(row), false);
270   }
271 
272   /**
273    * Finds the region on which the given row is being served.
274    * @param row Row to find.
275    * @return Location of the row.
276    * @throws IOException if a remote or network exception occurs
277    */
278   public HRegionLocation getRegionLocation(final byte [] row)
279   throws IOException {
280     return connection.getRegionLocation(tableName, row, false);
281   }
282 
283   @Override
284   public byte [] getTableName() {
285     return this.tableName;
286   }
287 
288   /**
289    * <em>INTERNAL</em> Used by unit tests and tools to do low-level
290    * manipulations.
291    * @return An HConnection instance.
292    */
293   // TODO(tsuna): Remove this.  Unit tests shouldn't require public helpers.
294   public HConnection getConnection() {
295     return this.connection;
296   }
297 
298   /**
299    * Gets the number of rows that a scanner will fetch at once.
300    * <p>
301    * The default value comes from {@code hbase.client.scanner.caching}.
302    */
303   public int getScannerCaching() {
304     return scannerCaching;
305   }
306 
307   /**
308    * Sets the number of rows that a scanner will fetch at once.
309    * <p>
310    * This will override the value specified by
311    * {@code hbase.client.scanner.caching}.
312    * Increasing this value will reduce the amount of work needed each time
313    * {@code next()} is called on a scanner, at the expense of memory use
314    * (since more rows will need to be maintained in memory by the scanners).
315    * @param scannerCaching the number of rows a scanner will fetch at once.
316    */
317   public void setScannerCaching(int scannerCaching) {
318     this.scannerCaching = scannerCaching;
319   }
320 
321   @Override
322   public HTableDescriptor getTableDescriptor() throws IOException {
323     return new UnmodifyableHTableDescriptor(
324       this.connection.getHTableDescriptor(this.tableName));
325   }
326 
327   /**
328    * Gets the starting row key for every region in the currently open table.
329    * <p>
330    * This is mainly useful for the MapReduce integration.
331    * @return Array of region starting row keys
332    * @throws IOException if a remote or network exception occurs
333    */
334   public byte [][] getStartKeys() throws IOException {
335     return getStartEndKeys().getFirst();
336   }
337 
338   /**
339    * Gets the ending row key for every region in the currently open table.
340    * <p>
341    * This is mainly useful for the MapReduce integration.
342    * @return Array of region ending row keys
343    * @throws IOException if a remote or network exception occurs
344    */
345   public byte[][] getEndKeys() throws IOException {
346     return getStartEndKeys().getSecond();
347   }
348 
349   /**
350    * Gets the starting and ending row keys for every region in the currently
351    * open table.
352    * <p>
353    * This is mainly useful for the MapReduce integration.
354    * @return Pair of arrays of region starting and ending row keys
355    * @throws IOException if a remote or network exception occurs
356    */
357   @SuppressWarnings("unchecked")
358   public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException {
359     final List<byte[]> startKeyList = new ArrayList<byte[]>();
360     final List<byte[]> endKeyList = new ArrayList<byte[]>();
361     MetaScannerVisitor visitor = new MetaScannerVisitor() {
362       public boolean processRow(Result rowResult) throws IOException {
363         byte [] bytes = rowResult.getValue(HConstants.CATALOG_FAMILY,
364           HConstants.REGIONINFO_QUALIFIER);
365         if (bytes == null) {
366           LOG.warn("Null " + HConstants.REGIONINFO_QUALIFIER + " cell in " +
367             rowResult);
368           return true;
369         }
370         HRegionInfo info = Writables.getHRegionInfo(bytes);
371         if (Bytes.equals(info.getTableDesc().getName(), getTableName())) {
372           if (!(info.isOffline() || info.isSplit())) {
373             startKeyList.add(info.getStartKey());
374             endKeyList.add(info.getEndKey());
375           }
376         }
377         return true;
378       }
379     };
380     MetaScanner.metaScan(configuration, visitor, this.tableName);
381     return new Pair(startKeyList.toArray(new byte[startKeyList.size()][]),
382                 endKeyList.toArray(new byte[endKeyList.size()][]));
383   }
384 
385   /**
386    * Gets all the regions and their address for this table.
387    * <p>
388    * This is mainly useful for the MapReduce integration.
389    * @return A map of HRegionInfo with it's server address
390    * @throws IOException if a remote or network exception occurs
391    */
392   public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
393     final Map<HRegionInfo, HServerAddress> regionMap =
394       new TreeMap<HRegionInfo, HServerAddress>();
395 
396     MetaScannerVisitor visitor = new MetaScannerVisitor() {
397       public boolean processRow(Result rowResult) throws IOException {
398         HRegionInfo info = Writables.getHRegionInfo(
399             rowResult.getValue(HConstants.CATALOG_FAMILY,
400                 HConstants.REGIONINFO_QUALIFIER));
401 
402         if (!(Bytes.equals(info.getTableDesc().getName(), getTableName()))) {
403           return false;
404         }
405 
406         HServerAddress server = new HServerAddress();
407         byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
408             HConstants.SERVER_QUALIFIER);
409         if (value != null && value.length > 0) {
410           String address = Bytes.toString(value);
411           server = new HServerAddress(address);
412         }
413 
414         if (!(info.isOffline() || info.isSplit())) {
415           regionMap.put(new UnmodifyableHRegionInfo(info), server);
416         }
417         return true;
418       }
419 
420     };
421     MetaScanner.metaScan(configuration, visitor, tableName);
422     return regionMap;
423   }
424 
425   /**
426    * Save the passed region information and the table's regions
427    * cache.
428    * <p>
429    * This is mainly useful for the MapReduce integration. You can call
430    * {@link #deserializeRegionInfo deserializeRegionInfo}
431    * to deserialize regions information from a
432    * {@link DataInput}, then call this method to load them to cache.
433    *
434    * <pre>
435    * {@code
436    * HTable t1 = new HTable("foo");
437    * FileInputStream fis = new FileInputStream("regions.dat");
438    * DataInputStream dis = new DataInputStream(fis);
439    *
440    * Map<HRegionInfo, HServerAddress> hm = t1.deserializeRegionInfo(dis);
441    * t1.prewarmRegionCache(hm);
442    * }
443    * </pre>
444    * @param regionMap This piece of regions information will be loaded
445    * to region cache.
446    */
447   public void prewarmRegionCache(Map<HRegionInfo, HServerAddress> regionMap) {
448     this.connection.prewarmRegionCache(this.getTableName(), regionMap);
449   }
450 
451   /**
452    * Serialize the regions information of this table and output
453    * to <code>out</code>.
454    * <p>
455    * This is mainly useful for the MapReduce integration. A client could
456    * perform a large scan for all the regions for the table, serialize the
457    * region info to a file. MR job can ship a copy of the meta for the table in
458    * the DistributedCache.
459    * <pre>
460    * {@code
461    * FileOutputStream fos = new FileOutputStream("regions.dat");
462    * DataOutputStream dos = new DataOutputStream(fos);
463    * table.serializeRegionInfo(dos);
464    * dos.flush();
465    * dos.close();
466    * }
467    * </pre>
468    * @param out {@link DataOutput} to serialize this object into.
469    * @throws IOException if a remote or network exception occurs
470    */
471   public void serializeRegionInfo(DataOutput out) throws IOException {
472     Map<HRegionInfo, HServerAddress> allRegions = this.getRegionsInfo();
473     // first, write number of regions
474     out.writeInt(allRegions.size());
475     for (Map.Entry<HRegionInfo, HServerAddress> es : allRegions.entrySet()) {
476       es.getKey().write(out);
477       es.getValue().write(out);
478     }
479   }
480 
481   /**
482    * Read from <code>in</code> and deserialize the regions information.
483    *
484    * <p>It behaves similarly as {@link #getRegionsInfo getRegionsInfo}, except
485    * that it loads the region map from a {@link DataInput} object.
486    *
487    * <p>It is supposed to be followed immediately by  {@link
488    * #prewarmRegionCache prewarmRegionCache}.
489    *
490    * <p>
491    * Please refer to {@link #prewarmRegionCache prewarmRegionCache} for usage.
492    *
493    * @param in {@link DataInput} object.
494    * @return A map of HRegionInfo with its server address.
495    * @throws IOException if an I/O exception occurs.
496    */
497   public Map<HRegionInfo, HServerAddress> deserializeRegionInfo(DataInput in)
498   throws IOException {
499     final Map<HRegionInfo, HServerAddress> allRegions =
500       new TreeMap<HRegionInfo, HServerAddress>();
501 
502     // the first integer is expected to be the size of records
503     int regionsCount = in.readInt();
504     for (int i = 0; i < regionsCount; ++i) {
505       HRegionInfo hri = new HRegionInfo();
506       hri.readFields(in);
507       HServerAddress hsa = new HServerAddress();
508       hsa.readFields(in);
509       allRegions.put(hri, hsa);
510     }
511     return allRegions;
512   }
513 
514    @Override
515    public Result getRowOrBefore(final byte[] row, final byte[] family)
516    throws IOException {
517      return connection.getRegionServerWithRetries(
518          new ServerCallable<Result>(connection, tableName, row) {
519        public Result call() throws IOException {
520          return server.getClosestRowBefore(location.getRegionInfo().getRegionName(),
521            row, family);
522        }
523      });
524    }
525 
526   @Override
527   public ResultScanner getScanner(final Scan scan) throws IOException {
528     ClientScanner s = new ClientScanner(scan);
529     s.initialize();
530     return s;
531   }
532 
533   @Override
534   public ResultScanner getScanner(byte [] family) throws IOException {
535     Scan scan = new Scan();
536     scan.addFamily(family);
537     return getScanner(scan);
538   }
539 
540   @Override
541   public ResultScanner getScanner(byte [] family, byte [] qualifier)
542   throws IOException {
543     Scan scan = new Scan();
544     scan.addColumn(family, qualifier);
545     return getScanner(scan);
546   }
547 
548   public Result get(final Get get) throws IOException {
549     return connection.getRegionServerWithRetries(
550         new ServerCallable<Result>(connection, tableName, get.getRow()) {
551           public Result call() throws IOException {
552             return server.get(location.getRegionInfo().getRegionName(), get);
553           }
554         }
555     );
556   }
557 
558    public Result[] get(List<Get> gets) throws IOException {
559      try {
560        Object [] r1 = batch((List)gets);
561 
562        // translate.
563        Result [] results = new Result[r1.length];
564        int i=0;
565        for (Object o : r1) {
566          // batch ensures if there is a failure we get an exception instead
567          results[i++] = (Result) o;
568        }
569 
570        return results;
571      } catch (InterruptedException e) {
572        throw new IOException(e);
573      }
574    }
575 
576   /**
577    * Method that does a batch call on Deletes, Gets and Puts.  The ordering of
578    * execution of the actions is not defined. Meaning if you do a Put and a
579    * Get in the same {@link #batch} call, you will not necessarily be
580    * guaranteed that the Get returns what the Put had put.
581    *
582    * @param actions list of Get, Put, Delete objects
583    * @param results Empty Result[], same size as actions. Provides access to
584    * partial results, in case an exception is thrown. If there are any failures,
585    * there will be a null or Throwable will be in the results array, AND an
586    * exception will be thrown.
587    * @throws IOException
588    */
589   @Override
590   public synchronized void batch(final List<Row> actions, final Object[] results)
591       throws InterruptedException, IOException {
592     connection.processBatch(actions, tableName, pool, results);
593   }
594 
595   /**
596    * Method that does a batch call on Deletes, Gets and Puts.
597    *
598    * @param actions list of Get, Put, Delete objects
599    * @return the results from the actions. A null in the return array means that
600    * the call for that action failed, even after retries
601    * @throws IOException
602    */
603   @Override
604   public synchronized Object[] batch(final List<Row> actions) throws InterruptedException, IOException {
605     Object[] results = new Object[actions.size()];
606     connection.processBatch(actions, tableName, pool, results);
607     return results;
608   }
609 
610   /**
611    * Deletes the specified cells/row.
612    *
613    * @param delete The object that specifies what to delete.
614    * @throws IOException if a remote or network exception occurs.
615    * @since 0.20.0
616    */
617   @Override
618   public void delete(final Delete delete)
619   throws IOException {
620     connection.getRegionServerWithRetries(
621         new ServerCallable<Boolean>(connection, tableName, delete.getRow()) {
622           public Boolean call() throws IOException {
623             server.delete(location.getRegionInfo().getRegionName(), delete);
624             return null; // FindBugs NP_BOOLEAN_RETURN_NULL
625           }
626         }
627     );
628   }
629 
630   /**
631    * Deletes the specified cells/rows in bulk.
632    * @param deletes List of things to delete. As a side effect, it will be modified:
633    * successful {@link Delete}s are removed. The ordering of the list will not change.
634    * @throws IOException if a remote or network exception occurs. In that case
635    * the {@code deletes} argument will contain the {@link Delete} instances
636    * that have not be successfully applied.
637    * @since 0.20.1
638    * @see #batch(java.util.List, Object[])
639    */
640   @Override
641   public void delete(final List<Delete> deletes)
642   throws IOException {
643     Object[] results = new Object[deletes.size()];
644     try {
645       connection.processBatch((List) deletes, tableName, pool, results);
646     } catch (InterruptedException e) {
647       throw new IOException(e);
648     } finally {
649       // mutate list so that it is empty for complete success, or contains only failed records
650       // results are returned in the same order as the requests in list
651       // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
652       for (int i = results.length - 1; i>=0; i--) {
653         // if result is not null, it succeeded
654         if (results[i] instanceof Result) {
655           deletes.remove(i);
656         }
657       }
658     }
659   }
660 
661   @Override
662   public void put(final Put put) throws IOException {
663     doPut(Arrays.asList(put));
664   }
665 
666   @Override
667   public void put(final List<Put> puts) throws IOException {
668     doPut(puts);
669   }
670 
671   private void doPut(final List<Put> puts) throws IOException {
672     for (Put put : puts) {
673       validatePut(put);
674       writeBuffer.add(put);
675       currentWriteBufferSize += put.heapSize();
676     }
677     if (autoFlush || currentWriteBufferSize > writeBufferSize) {
678       flushCommits();
679     }
680   }
681 
682   @Override
683   public Result increment(final Increment increment) throws IOException {
684     if (!increment.hasFamilies()) {
685       throw new IOException(
686           "Invalid arguments to increment, no columns specified");
687     }
688     return connection.getRegionServerWithRetries(
689         new ServerCallable<Result>(connection, tableName, increment.getRow()) {
690           public Result call() throws IOException {
691             return server.increment(
692                 location.getRegionInfo().getRegionName(), increment);
693           }
694         }
695     );
696   }
697 
698   @Override
699   public long incrementColumnValue(final byte [] row, final byte [] family,
700       final byte [] qualifier, final long amount)
701   throws IOException {
702     return incrementColumnValue(row, family, qualifier, amount, true);
703   }
704 
705   @Override
706   public long incrementColumnValue(final byte [] row, final byte [] family,
707       final byte [] qualifier, final long amount, final boolean writeToWAL)
708   throws IOException {
709     NullPointerException npe = null;
710     if (row == null) {
711       npe = new NullPointerException("row is null");
712     } else if (family == null) {
713       npe = new NullPointerException("column is null");
714     }
715     if (npe != null) {
716       throw new IOException(
717           "Invalid arguments to incrementColumnValue", npe);
718     }
719     return connection.getRegionServerWithRetries(
720         new ServerCallable<Long>(connection, tableName, row) {
721           public Long call() throws IOException {
722             return server.incrementColumnValue(
723                 location.getRegionInfo().getRegionName(), row, family,
724                 qualifier, amount, writeToWAL);
725           }
726         }
727     );
728   }
729 
730   /**
731    * Atomically checks if a row/family/qualifier value match the expectedValue.
732    * If it does, it adds the put.  If value == null, checks for non-existence
733    * of the value.
734    *
735    * @param row to check
736    * @param family column family
737    * @param qualifier column qualifier
738    * @param value the expected value
739    * @param put put to execute if value matches.
740    * @throws IOException
741    * @return true if the new put was execute, false otherwise
742    */
743   @Override
744   public boolean checkAndPut(final byte [] row,
745       final byte [] family, final byte [] qualifier, final byte [] value,
746       final Put put)
747   throws IOException {
748     return connection.getRegionServerWithRetries(
749         new ServerCallable<Boolean>(connection, tableName, row) {
750           public Boolean call() throws IOException {
751             return server.checkAndPut(location.getRegionInfo().getRegionName(),
752                 row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE;
753           }
754         }
755     );
756   }
757 
758   /**
759    * Atomically checks if a row/family/qualifier value match the expectedValue.
760    * If it does, it adds the delete.  If value == null, checks for non-existence
761    * of the value.
762    *
763    * @param row to check
764    * @param family column family
765    * @param qualifier column qualifier
766    * @param value the expected value
767    * @param delete delete to execute if value matches.
768    * @throws IOException
769    * @return true if the new delete was executed, false otherwise
770    */
771   @Override
772   public boolean checkAndDelete(final byte [] row,
773       final byte [] family, final byte [] qualifier, final byte [] value,
774       final Delete delete)
775   throws IOException {
776     return connection.getRegionServerWithRetries(
777         new ServerCallable<Boolean>(connection, tableName, row) {
778           public Boolean call() throws IOException {
779             return server.checkAndDelete(
780                 location.getRegionInfo().getRegionName(),
781                 row, family, qualifier, value, delete)
782             ? Boolean.TRUE : Boolean.FALSE;
783           }
784         }
785     );
786   }
787 
788   /**
789    * Test for the existence of columns in the table, as specified in the Get.<p>
790    *
791    * This will return true if the Get matches one or more keys, false if not.<p>
792    *
793    * This is a server-side call so it prevents any data from being transfered
794    * to the client.
795    * @param get param to check for
796    * @return true if the specified Get matches one or more keys, false if not
797    * @throws IOException
798    */
799   @Override
800   public boolean exists(final Get get) throws IOException {
801     return connection.getRegionServerWithRetries(
802         new ServerCallable<Boolean>(connection, tableName, get.getRow()) {
803           public Boolean call() throws IOException {
804             return server.
805                 exists(location.getRegionInfo().getRegionName(), get);
806           }
807         }
808     );
809   }
810 
811   /**
812    * Executes all the buffered {@link Put} operations.
813    * <p>
814    * This method gets called once automatically for every {@link Put} or batch
815    * of {@link Put}s (when {@link #batch(List)} is used) when
816    * {@link #isAutoFlush()} is {@code true}.
817    * @throws IOException if a remote or network exception occurs.
818    */
819   @Override
820   public void flushCommits() throws IOException {
821     try {
822       connection.processBatchOfPuts(writeBuffer, tableName, pool);
823     } finally {
824       // the write buffer was adjusted by processBatchOfPuts
825       currentWriteBufferSize = 0;
826       for (Put aPut : writeBuffer) {
827         currentWriteBufferSize += aPut.heapSize();
828       }
829     }
830   }
831 
832   @Override
833   public void close() throws IOException {
834     flushCommits();
835   }
836 
837   // validate for well-formedness
838   private void validatePut(final Put put) throws IllegalArgumentException{
839     if (put.isEmpty()) {
840       throw new IllegalArgumentException("No columns to insert");
841     }
842     if (maxKeyValueSize > 0) {
843       for (List<KeyValue> list : put.getFamilyMap().values()) {
844         for (KeyValue kv : list) {
845           if (kv.getLength() > maxKeyValueSize) {
846             throw new IllegalArgumentException("KeyValue size too large");
847           }
848         }
849       }
850     }
851   }
852 
853   @Override
854   public RowLock lockRow(final byte [] row)
855   throws IOException {
856     return connection.getRegionServerWithRetries(
857       new ServerCallable<RowLock>(connection, tableName, row) {
858         public RowLock call() throws IOException {
859           long lockId =
860               server.lockRow(location.getRegionInfo().getRegionName(), row);
861           return new RowLock(row,lockId);
862         }
863       }
864     );
865   }
866 
867   @Override
868   public void unlockRow(final RowLock rl)
869   throws IOException {
870     connection.getRegionServerWithRetries(
871       new ServerCallable<Boolean>(connection, tableName, rl.getRow()) {
872         public Boolean call() throws IOException {
873           server.unlockRow(location.getRegionInfo().getRegionName(),
874               rl.getLockId());
875           return null; // FindBugs NP_BOOLEAN_RETURN_NULL
876         }
877       }
878     );
879   }
880 
881   /**
882    * Explicitly clears the region cache to fetch the latest value from META.
883    * This is a power user function: avoid unless you know the ramifications.
884    */
885   public void clearRegionCache() {
886     this.connection.clearRegionCache();
887   }
888 
889   @Override
890   public boolean isAutoFlush() {
891     return autoFlush;
892   }
893 
894   /**
895    * Turns 'auto-flush' on or off.
896    * <p>
897    * When enabled (default), {@link Put} operations don't get buffered/delayed
898    * and are immediately executed.  This is slower but safer.
899    * <p>
900    * Turning this off means that multiple {@link Put}s will be accepted before
901    * any RPC is actually sent to do the write operations.  If the application
902    * dies before pending writes get flushed to HBase, data will be lost.
903    * Other side effects may include the fact that the application thinks a
904    * {@link Put} was executed successfully whereas it was in fact only
905    * buffered and the operation may fail when attempting to flush all pending
906    * writes.  In that case though, the code will retry the failed {@link Put}
907    * upon its next attempt to flush the buffer.
908    *
909    * @param autoFlush Whether or not to enable 'auto-flush'.
910    * @see #flushCommits
911    */
912   public void setAutoFlush(boolean autoFlush) {
913     this.autoFlush = autoFlush;
914   }
915 
916   /**
917    * Returns the maximum size in bytes of the write buffer for this HTable.
918    * <p>
919    * The default value comes from the configuration parameter
920    * {@code hbase.client.write.buffer}.
921    * @return The size of the write buffer in bytes.
922    */
923   public long getWriteBufferSize() {
924     return writeBufferSize;
925   }
926 
927   /**
928    * Sets the size of the buffer in bytes.
929    * <p>
930    * If the new size is less than the current amount of data in the
931    * write buffer, the buffer gets flushed.
932    * @param writeBufferSize The new write buffer size, in bytes.
933    * @throws IOException if a remote or network exception occurs.
934    */
935   public void setWriteBufferSize(long writeBufferSize) throws IOException {
936     this.writeBufferSize = writeBufferSize;
937     if(currentWriteBufferSize > writeBufferSize) {
938       flushCommits();
939     }
940   }
941 
942   /**
943    * Returns the write buffer.
944    * @return The current write buffer.
945    */
946   public ArrayList<Put> getWriteBuffer() {
947     return writeBuffer;
948   }
949 
950   /**
951    * Implements the scanner interface for the HBase client.
952    * If there are multiple regions in a table, this scanner will iterate
953    * through them all.
954    */
955   protected class ClientScanner implements ResultScanner {
956     private final Log CLIENT_LOG = LogFactory.getLog(this.getClass());
957     // HEADSUP: The scan internal start row can change as we move through table.
958     private Scan scan;
959     private boolean closed = false;
960     // Current region scanner is against.  Gets cleared if current region goes
961     // wonky: e.g. if it splits on us.
962     private HRegionInfo currentRegion = null;
963     private ScannerCallable callable = null;
964     private final LinkedList<Result> cache = new LinkedList<Result>();
965     private final int caching;
966     private long lastNext;
967     // Keep lastResult returned successfully in case we have to reset scanner.
968     private Result lastResult = null;
969 
970     protected ClientScanner(final Scan scan) {
971       if (CLIENT_LOG.isDebugEnabled()) {
972         CLIENT_LOG.debug("Creating scanner over "
973             + Bytes.toString(getTableName())
974             + " starting at key '" + Bytes.toStringBinary(scan.getStartRow()) + "'");
975       }
976       this.scan = scan;
977       this.lastNext = System.currentTimeMillis();
978 
979       // Use the caching from the Scan.  If not set, use the default cache setting for this table.
980       if (this.scan.getCaching() > 0) {
981         this.caching = this.scan.getCaching();
982       } else {
983         this.caching = HTable.this.scannerCaching;
984       }
985 
986       // Removed filter validation.  We have a new format now, only one of all
987       // the current filters has a validate() method.  We can add it back,
988       // need to decide on what we're going to do re: filter redesign.
989       // Need, at the least, to break up family from qualifier as separate
990       // checks, I think it's important server-side filters are optimal in that
991       // respect.
992     }
993 
994     public void initialize() throws IOException {
995       nextScanner(this.caching, false);
996     }
997 
998     protected Scan getScan() {
999       return scan;
1000     }
1001 
1002     protected long getTimestamp() {
1003       return lastNext;
1004     }
1005 
1006     // returns true if the passed region endKey
1007     private boolean checkScanStopRow(final byte [] endKey) {
1008       if (this.scan.getStopRow().length > 0) {
1009         // there is a stop row, check to see if we are past it.
1010         byte [] stopRow = scan.getStopRow();
1011         int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
1012           endKey, 0, endKey.length);
1013         if (cmp <= 0) {
1014           // stopRow <= endKey (endKey is equals to or larger than stopRow)
1015           // This is a stop.
1016           return true;
1017         }
1018       }
1019       return false; //unlikely.
1020     }
1021 
1022     /*
1023      * Gets a scanner for the next region.  If this.currentRegion != null, then
1024      * we will move to the endrow of this.currentRegion.  Else we will get
1025      * scanner at the scan.getStartRow().  We will go no further, just tidy
1026      * up outstanding scanners, if <code>currentRegion != null</code> and
1027      * <code>done</code> is true.
1028      * @param nbRows
1029      * @param done Server-side says we're done scanning.
1030      */
1031     private boolean nextScanner(int nbRows, final boolean done)
1032     throws IOException {
1033       // Close the previous scanner if it's open
1034       if (this.callable != null) {
1035         this.callable.setClose();
1036         getConnection().getRegionServerWithRetries(callable);
1037         this.callable = null;
1038       }
1039 
1040       // Where to start the next scanner
1041       byte [] localStartKey;
1042 
1043       // if we're at end of table, close and return false to stop iterating
1044       if (this.currentRegion != null) {
1045         byte [] endKey = this.currentRegion.getEndKey();
1046         if (endKey == null ||
1047             Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
1048             checkScanStopRow(endKey) ||
1049             done) {
1050           close();
1051           if (CLIENT_LOG.isDebugEnabled()) {
1052             CLIENT_LOG.debug("Finished with scanning at " + this.currentRegion);
1053           }
1054           return false;
1055         }
1056         localStartKey = endKey;
1057         if (CLIENT_LOG.isDebugEnabled()) {
1058           CLIENT_LOG.debug("Finished with region " + this.currentRegion);
1059         }
1060       } else {
1061         localStartKey = this.scan.getStartRow();
1062       }
1063 
1064       if (CLIENT_LOG.isDebugEnabled()) {
1065         CLIENT_LOG.debug("Advancing internal scanner to startKey at '" +
1066           Bytes.toStringBinary(localStartKey) + "'");
1067       }
1068       try {
1069         callable = getScannerCallable(localStartKey, nbRows);
1070         // Open a scanner on the region server starting at the
1071         // beginning of the region
1072         getConnection().getRegionServerWithRetries(callable);
1073         this.currentRegion = callable.getHRegionInfo();
1074       } catch (IOException e) {
1075         close();
1076         throw e;
1077       }
1078       return true;
1079     }
1080 
1081     protected ScannerCallable getScannerCallable(byte [] localStartKey,
1082         int nbRows) {
1083       scan.setStartRow(localStartKey);
1084       ScannerCallable s = new ScannerCallable(getConnection(),
1085         getTableName(), scan);
1086       s.setCaching(nbRows);
1087       return s;
1088     }
1089 
1090     public Result next() throws IOException {
1091       // If the scanner is closed but there is some rows left in the cache,
1092       // it will first empty it before returning null
1093       if (cache.size() == 0 && this.closed) {
1094         return null;
1095       }
1096       if (cache.size() == 0) {
1097         Result [] values = null;
1098         long remainingResultSize = maxScannerResultSize;
1099         int countdown = this.caching;
1100         // We need to reset it if it's a new callable that was created
1101         // with a countdown in nextScanner
1102         callable.setCaching(this.caching);
1103         // This flag is set when we want to skip the result returned.  We do
1104         // this when we reset scanner because it split under us.
1105         boolean skipFirst = false;
1106         do {
1107           try {
1108             if (skipFirst) {
1109               // Skip only the first row (which was the last row of the last
1110               // already-processed batch).
1111               callable.setCaching(1);
1112               values = getConnection().getRegionServerWithRetries(callable);
1113               callable.setCaching(this.caching);
1114               skipFirst = false;
1115             }
1116             // Server returns a null values if scanning is to stop.  Else,
1117             // returns an empty array if scanning is to go on and we've just
1118             // exhausted current region.
1119             values = getConnection().getRegionServerWithRetries(callable);
1120           } catch (DoNotRetryIOException e) {
1121             if (e instanceof UnknownScannerException) {
1122               long timeout = lastNext + scannerTimeout;
1123               // If we are over the timeout, throw this exception to the client
1124               // Else, it's because the region moved and we used the old id
1125               // against the new region server; reset the scanner.
1126               if (timeout < System.currentTimeMillis()) {
1127                 long elapsed = System.currentTimeMillis() - lastNext;
1128                 ScannerTimeoutException ex = new ScannerTimeoutException(
1129                     elapsed + "ms passed since the last invocation, " +
1130                         "timeout is currently set to " + scannerTimeout);
1131                 ex.initCause(e);
1132                 throw ex;
1133               }
1134             } else {
1135               Throwable cause = e.getCause();
1136               if (cause == null || !(cause instanceof NotServingRegionException)) {
1137                 throw e;
1138               }
1139             }
1140             // Else, its signal from depths of ScannerCallable that we got an
1141             // NSRE on a next and that we need to reset the scanner.
1142             if (this.lastResult != null) {
1143               this.scan.setStartRow(this.lastResult.getRow());
1144               // Skip first row returned.  We already let it out on previous
1145               // invocation.
1146               skipFirst = true;
1147             }
1148             // Clear region
1149             this.currentRegion = null;
1150             continue;
1151           }
1152           lastNext = System.currentTimeMillis();
1153           if (values != null && values.length > 0) {
1154             for (Result rs : values) {
1155               cache.add(rs);
1156               for (KeyValue kv : rs.raw()) {
1157                   remainingResultSize -= kv.heapSize();
1158               }
1159               countdown--;
1160               this.lastResult = rs;
1161             }
1162           }
1163           // Values == null means server-side filter has determined we must STOP
1164         } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
1165       }
1166 
1167       if (cache.size() > 0) {
1168         return cache.poll();
1169       }
1170       return null;
1171     }
1172 
1173     /**
1174      * Get <param>nbRows</param> rows.
1175      * How many RPCs are made is determined by the {@link Scan#setCaching(int)}
1176      * setting (or hbase.client.scanner.caching in hbase-site.xml).
1177      * @param nbRows number of rows to return
1178      * @return Between zero and <param>nbRows</param> RowResults.  Scan is done
1179      * if returned array is of zero-length (We never return null).
1180      * @throws IOException
1181      */
1182     public Result [] next(int nbRows) throws IOException {
1183       // Collect values to be returned here
1184       ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
1185       for(int i = 0; i < nbRows; i++) {
1186         Result next = next();
1187         if (next != null) {
1188           resultSets.add(next);
1189         } else {
1190           break;
1191         }
1192       }
1193       return resultSets.toArray(new Result[resultSets.size()]);
1194     }
1195 
1196     public void close() {
1197       if (callable != null) {
1198         callable.setClose();
1199         try {
1200           getConnection().getRegionServerWithRetries(callable);
1201         } catch (IOException e) {
1202           // We used to catch this error, interpret, and rethrow. However, we
1203           // have since decided that it's not nice for a scanner's close to
1204           // throw exceptions. Chances are it was just an UnknownScanner
1205           // exception due to lease time out.
1206         }
1207         callable = null;
1208       }
1209       closed = true;
1210     }
1211 
1212     public Iterator<Result> iterator() {
1213       return new Iterator<Result>() {
1214         // The next RowResult, possibly pre-read
1215         Result next = null;
1216 
1217         // return true if there is another item pending, false if there isn't.
1218         // this method is where the actual advancing takes place, but you need
1219         // to call next() to consume it. hasNext() will only advance if there
1220         // isn't a pending next().
1221         public boolean hasNext() {
1222           if (next == null) {
1223             try {
1224               next = ClientScanner.this.next();
1225               return next != null;
1226             } catch (IOException e) {
1227               throw new RuntimeException(e);
1228             }
1229           }
1230           return true;
1231         }
1232 
1233         // get the pending next item and advance the iterator. returns null if
1234         // there is no next item.
1235         public Result next() {
1236           // since hasNext() does the real advancing, we call this to determine
1237           // if there is a next before proceeding.
1238           if (!hasNext()) {
1239             return null;
1240           }
1241 
1242           // if we get to here, then hasNext() has given us an item to return.
1243           // we want to return the item and then null out the next pointer, so
1244           // we use a temporary variable.
1245           Result temp = next;
1246           next = null;
1247           return temp;
1248         }
1249 
1250         public void remove() {
1251           throw new UnsupportedOperationException();
1252         }
1253       };
1254     }
1255   }
1256 
1257   static class DaemonThreadFactory implements ThreadFactory {
1258     static final AtomicInteger poolNumber = new AtomicInteger(1);
1259         final ThreadGroup group;
1260         final AtomicInteger threadNumber = new AtomicInteger(1);
1261         final String namePrefix;
1262 
1263         DaemonThreadFactory() {
1264             SecurityManager s = System.getSecurityManager();
1265             group = (s != null)? s.getThreadGroup() :
1266                                  Thread.currentThread().getThreadGroup();
1267             namePrefix = "pool-" +
1268                           poolNumber.getAndIncrement() +
1269                          "-thread-";
1270         }
1271 
1272         public Thread newThread(Runnable r) {
1273             Thread t = new Thread(group, r,
1274                                   namePrefix + threadNumber.getAndIncrement(),
1275                                   0);
1276             if (!t.isDaemon()) {
1277               t.setDaemon(true);
1278             }
1279             if (t.getPriority() != Thread.NORM_PRIORITY) {
1280               t.setPriority(Thread.NORM_PRIORITY);
1281             }
1282             return t;
1283         }
1284   }
1285 
1286   /**
1287    * Enable or disable region cache prefetch for the table. It will be
1288    * applied for the given table's all HTable instances who share the same
1289    * connection. By default, the cache prefetch is enabled.
1290    * @param tableName name of table to configure.
1291    * @param enable Set to true to enable region cache prefetch. Or set to
1292    * false to disable it.
1293    * @throws ZooKeeperConnectionException
1294    */
1295   public static void setRegionCachePrefetch(final byte[] tableName,
1296       boolean enable) throws ZooKeeperConnectionException {
1297     HConnectionManager.getConnection(HBaseConfiguration.create()).
1298     setRegionCachePrefetch(tableName, enable);
1299   }
1300 
1301   /**
1302    * Enable or disable region cache prefetch for the table. It will be
1303    * applied for the given table's all HTable instances who share the same
1304    * connection. By default, the cache prefetch is enabled.
1305    * @param conf The Configuration object to use.
1306    * @param tableName name of table to configure.
1307    * @param enable Set to true to enable region cache prefetch. Or set to
1308    * false to disable it.
1309    * @throws ZooKeeperConnectionException
1310    */
1311   public static void setRegionCachePrefetch(final Configuration conf,
1312       final byte[] tableName, boolean enable) throws ZooKeeperConnectionException {
1313     HConnectionManager.getConnection(conf).setRegionCachePrefetch(
1314         tableName, enable);
1315   }
1316 
1317   /**
1318    * Check whether region cache prefetch is enabled or not for the table.
1319    * @param conf The Configuration object to use.
1320    * @param tableName name of table to check
1321    * @return true if table's region cache prefecth is enabled. Otherwise
1322    * it is disabled.
1323    * @throws ZooKeeperConnectionException
1324    */
1325   public static boolean getRegionCachePrefetch(final Configuration conf,
1326       final byte[] tableName) throws ZooKeeperConnectionException {
1327     return HConnectionManager.getConnection(conf).getRegionCachePrefetch(
1328         tableName);
1329   }
1330 
1331   /**
1332    * Check whether region cache prefetch is enabled or not for the table.
1333    * @param tableName name of table to check
1334    * @return true if table's region cache prefecth is enabled. Otherwise
1335    * it is disabled.
1336    * @throws ZooKeeperConnectionException
1337    */
1338   public static boolean getRegionCachePrefetch(final byte[] tableName) throws ZooKeeperConnectionException {
1339     return HConnectionManager.getConnection(HBaseConfiguration.create()).
1340     getRegionCachePrefetch(tableName);
1341   }
1342 }