View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.Iterator;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.TreeMap;
32  import java.util.concurrent.ExecutorService;
33  import java.util.concurrent.LinkedBlockingQueue;
34  import java.util.concurrent.ThreadFactory;
35  import java.util.concurrent.ThreadPoolExecutor;
36  import java.util.concurrent.TimeUnit;
37  import java.util.concurrent.atomic.AtomicInteger;
38  
39  import org.apache.commons.logging.Log;
40  import org.apache.commons.logging.LogFactory;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.DoNotRetryIOException;
43  import org.apache.hadoop.hbase.HBaseConfiguration;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HRegionLocation;
47  import org.apache.hadoop.hbase.HServerAddress;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.KeyValue;
50  import org.apache.hadoop.hbase.NotServingRegionException;
51  import org.apache.hadoop.hbase.UnknownScannerException;
52  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
53  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
54  import org.apache.hadoop.hbase.util.Bytes;
55  import org.apache.hadoop.hbase.util.Pair;
56  import org.apache.hadoop.hbase.util.Writables;
57  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
58  import org.apache.zookeeper.KeeperException;
59  
60  /**
61   * Used to communicate with a single HBase table.
62   *
63   * This class is not thread safe for updates; the underlying write buffer can
64   * be corrupted if multiple threads contend over a single HTable instance.
65   *
66   * <p>Instances of HTable passed the same {@link Configuration} instance will
67   * share connections to servers out on the cluster and to the zookeeper ensemble
68   * as well as caches of  region locations.  This is usually a *good* thing.
69   * This happens because they will all share the same underlying
70   * {@link HConnection} instance.  See {@link HConnectionManager} for more on
71   * how this mechanism works.
72   *
73   * <p>{@link HConnection} will read most of the
74   * configuration it needs from the passed {@link Configuration} on initial
75   * construction.  Thereafter, for settings such as
76   * <code>hbase.client.pause</code>, <code>hbase.client.retries.number</code>,
77   * and <code>hbase.client.rpc.maxattempts</code> updating their values in the
78   * passed {@link Configuration} subsequent to {@link HConnection} construction
79   * will go unnoticed.  To run with changed values, make a new
80   * {@link HTable} passing a new {@link Configuration} instance that has the
81   * new configuration.
82   *
83   * @see HBaseAdmin for create, drop, list, enable and disable of tables.
84   * @see HConnection
85   * @see HConnectionManager
86   */
87  public class HTable implements HTableInterface {
88    private static final Log LOG = LogFactory.getLog(HTable.class);
89    private final HConnection connection;
90    private final byte [] tableName;
91    protected final int scannerTimeout;
92    private volatile Configuration configuration;
93    private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
94    private long writeBufferSize;
95    private boolean autoFlush;
96    private long currentWriteBufferSize;
97    protected int scannerCaching;
98    private int maxKeyValueSize;
99    private ExecutorService pool;  // For Multi
100   private long maxScannerResultSize;
101 
102   /**
103    * Creates an object to access a HBase table.
104    * Internally it creates a new instance of {@link Configuration} and a new
105    * client to zookeeper as well as other resources.  It also comes up with
106    * a fresh view of the cluster and must do discovery from scratch of region
107    * locations; i.e. it will not make use of already-cached region locations if
108    * available. Use only when being quick and dirty.
109    * @throws IOException if a remote or network exception occurs
110    * @see #HTable(Configuration, String)
111    */
112   public HTable(final String tableName)
113   throws IOException {
114     this(HBaseConfiguration.create(), Bytes.toBytes(tableName));
115   }
116 
117   /**
118    * Creates an object to access a HBase table.
119    * Internally it creates a new instance of {@link Configuration} and a new
120    * client to zookeeper as well as other resources.  It also comes up with
121    * a fresh view of the cluster and must do discovery from scratch of region
122    * locations; i.e. it will not make use of already-cached region locations if
123    * available. Use only when being quick and dirty.
124    * @param tableName Name of the table.
125    * @throws IOException if a remote or network exception occurs
126    * @see #HTable(Configuration, String)
127    */
128   public HTable(final byte [] tableName)
129   throws IOException {
130     this(HBaseConfiguration.create(), tableName);
131   }
132 
133   /**
134    * Creates an object to access a HBase table.
135    * Shares zookeeper connection and other resources with other HTable instances
136    * created with the same <code>conf</code> instance.  Uses already-populated
137    * region cache if one is available, populated by any other HTable instances
138    * sharing this <code>conf</code> instance.  Recommended.
139    * @param conf Configuration object to use.
140    * @param tableName Name of the table.
141    * @throws IOException if a remote or network exception occurs
142    */
143   public HTable(Configuration conf, final String tableName)
144   throws IOException {
145     this(conf, Bytes.toBytes(tableName));
146   }
147 
148 
149   /**
150    * Creates an object to access a HBase table.
151    * Shares zookeeper connection and other resources with other HTable instances
152    * created with the same <code>conf</code> instance.  Uses already-populated
153    * region cache if one is available, populated by any other HTable instances
154    * sharing this <code>conf</code> instance.  Recommended.
155    * @param conf Configuration object to use.
156    * @param tableName Name of the table.
157    * @throws IOException if a remote or network exception occurs
158    */
159   public HTable(Configuration conf, final byte [] tableName)
160   throws IOException {
161     this.tableName = tableName;
162     if (conf == null) {
163       this.scannerTimeout = 0;
164       this.connection = null;
165       return;
166     }
167     this.connection = HConnectionManager.getConnection(conf);
168     this.scannerTimeout =
169       (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
170     this.configuration = conf;
171     this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
172     this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152);
173     this.autoFlush = true;
174     this.currentWriteBufferSize = 0;
175     this.scannerCaching = conf.getInt("hbase.client.scanner.caching", 1);
176 
177     this.maxScannerResultSize = conf.getLong(
178       HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
179       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
180     this.maxKeyValueSize = conf.getInt("hbase.client.keyvalue.maxsize", -1);
181 
182     int nrThreads = conf.getInt("hbase.htable.threads.max", getCurrentNrHRS());
183     if (nrThreads == 0) {
184       nrThreads = 1; // is there a better default?
185     }
186 
187     // Unfortunately Executors.newCachedThreadPool does not allow us to
188     // set the maximum size of the pool, so we have to do it ourselves.
189     this.pool = new ThreadPoolExecutor(0, nrThreads,
190         60, TimeUnit.SECONDS,
191         new LinkedBlockingQueue<Runnable>(),
192         new DaemonThreadFactory());
193   }
194 
195   public Configuration getConfiguration() {
196     return configuration;
197   }
198 
199   /**
200    * @return the number of region servers that are currently running
201    * @throws IOException if a remote or network exception occurs
202    */
203   public int getCurrentNrHRS() throws IOException {
204     try {
205       // We go to zk rather than to master to get count of regions to avoid
206       // HTable having a Master dependency.  See HBase-2828
207       return ZKUtil.getNumberOfChildren(this.connection.getZooKeeperWatcher(),
208           this.connection.getZooKeeperWatcher().rsZNode);
209     } catch (KeeperException ke) {
210       throw new IOException("Unexpected ZooKeeper exception", ke);
211     }
212   }
213 
214   /**
215    * Tells whether or not a table is enabled or not.
216    * @param tableName Name of table to check.
217    * @return {@code true} if table is online.
218    * @throws IOException if a remote or network exception occurs
219    */
220   public static boolean isTableEnabled(String tableName) throws IOException {
221     return isTableEnabled(Bytes.toBytes(tableName));
222   }
223 
224   /**
225    * Tells whether or not a table is enabled or not.
226    * @param tableName Name of table to check.
227    * @return {@code true} if table is online.
228    * @throws IOException if a remote or network exception occurs
229    */
230   public static boolean isTableEnabled(byte[] tableName) throws IOException {
231     return isTableEnabled(HBaseConfiguration.create(), tableName);
232   }
233 
234   /**
235    * Tells whether or not a table is enabled or not.
236    * @param conf The Configuration object to use.
237    * @param tableName Name of table to check.
238    * @return {@code true} if table is online.
239    * @throws IOException if a remote or network exception occurs
240    */
241   public static boolean isTableEnabled(Configuration conf, String tableName)
242   throws IOException {
243     return isTableEnabled(conf, Bytes.toBytes(tableName));
244   }
245 
246   /**
247    * Tells whether or not a table is enabled or not.
248    * @param conf The Configuration object to use.
249    * @param tableName Name of table to check.
250    * @return {@code true} if table is online.
251    * @throws IOException if a remote or network exception occurs
252    */
253   public static boolean isTableEnabled(Configuration conf, byte[] tableName)
254   throws IOException {
255     return HConnectionManager.getConnection(conf).isTableEnabled(tableName);
256   }
257 
258   /**
259    * Find region location hosting passed row using cached info
260    * @param row Row to find.
261    * @return The location of the given row.
262    * @throws IOException if a remote or network exception occurs
263    */
264   public HRegionLocation getRegionLocation(final String row)
265   throws IOException {
266     return connection.getRegionLocation(tableName, Bytes.toBytes(row), false);
267   }
268 
269   /**
270    * Finds the region on which the given row is being served.
271    * @param row Row to find.
272    * @return Location of the row.
273    * @throws IOException if a remote or network exception occurs
274    */
275   public HRegionLocation getRegionLocation(final byte [] row)
276   throws IOException {
277     return connection.getRegionLocation(tableName, row, false);
278   }
279 
280   @Override
281   public byte [] getTableName() {
282     return this.tableName;
283   }
284 
285   /**
286    * <em>INTERNAL</em> Used by unit tests and tools to do low-level
287    * manipulations.
288    * @return An HConnection instance.
289    */
290   // TODO(tsuna): Remove this.  Unit tests shouldn't require public helpers.
291   public HConnection getConnection() {
292     return this.connection;
293   }
294 
295   /**
296    * Gets the number of rows that a scanner will fetch at once.
297    * <p>
298    * The default value comes from {@code hbase.client.scanner.caching}.
299    */
300   public int getScannerCaching() {
301     return scannerCaching;
302   }
303 
304   /**
305    * Sets the number of rows that a scanner will fetch at once.
306    * <p>
307    * This will override the value specified by
308    * {@code hbase.client.scanner.caching}.
309    * Increasing this value will reduce the amount of work needed each time
310    * {@code next()} is called on a scanner, at the expense of memory use
311    * (since more rows will need to be maintained in memory by the scanners).
312    * @param scannerCaching the number of rows a scanner will fetch at once.
313    */
314   public void setScannerCaching(int scannerCaching) {
315     this.scannerCaching = scannerCaching;
316   }
317 
318   @Override
319   public HTableDescriptor getTableDescriptor() throws IOException {
320     return new UnmodifyableHTableDescriptor(
321       this.connection.getHTableDescriptor(this.tableName));
322   }
323 
324   /**
325    * Gets the starting row key for every region in the currently open table.
326    * <p>
327    * This is mainly useful for the MapReduce integration.
328    * @return Array of region starting row keys
329    * @throws IOException if a remote or network exception occurs
330    */
331   public byte [][] getStartKeys() throws IOException {
332     return getStartEndKeys().getFirst();
333   }
334 
335   /**
336    * Gets the ending row key for every region in the currently open table.
337    * <p>
338    * This is mainly useful for the MapReduce integration.
339    * @return Array of region ending row keys
340    * @throws IOException if a remote or network exception occurs
341    */
342   public byte[][] getEndKeys() throws IOException {
343     return getStartEndKeys().getSecond();
344   }
345 
346   /**
347    * Gets the starting and ending row keys for every region in the currently
348    * open table.
349    * <p>
350    * This is mainly useful for the MapReduce integration.
351    * @return Pair of arrays of region starting and ending row keys
352    * @throws IOException if a remote or network exception occurs
353    */
354   @SuppressWarnings("unchecked")
355   public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException {
356     final List<byte[]> startKeyList = new ArrayList<byte[]>();
357     final List<byte[]> endKeyList = new ArrayList<byte[]>();
358     MetaScannerVisitor visitor = new MetaScannerVisitor() {
359       public boolean processRow(Result rowResult) throws IOException {
360         byte [] bytes = rowResult.getValue(HConstants.CATALOG_FAMILY,
361           HConstants.REGIONINFO_QUALIFIER);
362         if (bytes == null) {
363           LOG.warn("Null " + HConstants.REGIONINFO_QUALIFIER + " cell in " +
364             rowResult);
365           return true;
366         }
367         HRegionInfo info = Writables.getHRegionInfo(bytes);
368         if (Bytes.equals(info.getTableDesc().getName(), getTableName())) {
369           if (!(info.isOffline() || info.isSplit())) {
370             startKeyList.add(info.getStartKey());
371             endKeyList.add(info.getEndKey());
372           }
373         }
374         return true;
375       }
376     };
377     MetaScanner.metaScan(configuration, visitor, this.tableName);
378     return new Pair(startKeyList.toArray(new byte[startKeyList.size()][]),
379                 endKeyList.toArray(new byte[endKeyList.size()][]));
380   }
381 
382   /**
383    * Gets all the regions and their address for this table.
384    * <p>
385    * This is mainly useful for the MapReduce integration.
386    * @return A map of HRegionInfo with it's server address
387    * @throws IOException if a remote or network exception occurs
388    */
389   public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
390     final Map<HRegionInfo, HServerAddress> regionMap =
391       new TreeMap<HRegionInfo, HServerAddress>();
392 
393     MetaScannerVisitor visitor = new MetaScannerVisitor() {
394       public boolean processRow(Result rowResult) throws IOException {
395         HRegionInfo info = Writables.getHRegionInfo(
396             rowResult.getValue(HConstants.CATALOG_FAMILY,
397                 HConstants.REGIONINFO_QUALIFIER));
398 
399         if (!(Bytes.equals(info.getTableDesc().getName(), getTableName()))) {
400           return false;
401         }
402 
403         HServerAddress server = new HServerAddress();
404         byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
405             HConstants.SERVER_QUALIFIER);
406         if (value != null && value.length > 0) {
407           String address = Bytes.toString(value);
408           server = new HServerAddress(address);
409         }
410 
411         if (!(info.isOffline() || info.isSplit())) {
412           regionMap.put(new UnmodifyableHRegionInfo(info), server);
413         }
414         return true;
415       }
416 
417     };
418     MetaScanner.metaScan(configuration, visitor, tableName);
419     return regionMap;
420   }
421 
422   /**
423    * Save the passed region information and the table's regions
424    * cache.
425    * <p>
426    * This is mainly useful for the MapReduce integration. You can call
427    * {@link #deserializeRegionInfo deserializeRegionInfo}
428    * to deserialize regions information from a
429    * {@link DataInput}, then call this method to load them to cache.
430    *
431    * <pre>
432    * {@code
433    * HTable t1 = new HTable("foo");
434    * FileInputStream fis = new FileInputStream("regions.dat");
435    * DataInputStream dis = new DataInputStream(fis);
436    *
437    * Map<HRegionInfo, HServerAddress> hm = t1.deserializeRegionInfo(dis);
438    * t1.prewarmRegionCache(hm);
439    * }
440    * </pre>
441    * @param regionMap This piece of regions information will be loaded
442    * to region cache.
443    */
444   public void prewarmRegionCache(Map<HRegionInfo, HServerAddress> regionMap) {
445     this.connection.prewarmRegionCache(this.getTableName(), regionMap);
446   }
447 
448   /**
449    * Serialize the regions information of this table and output
450    * to <code>out</code>.
451    * <p>
452    * This is mainly useful for the MapReduce integration. A client could
453    * perform a large scan for all the regions for the table, serialize the
454    * region info to a file. MR job can ship a copy of the meta for the table in
455    * the DistributedCache.
456    * <pre>
457    * {@code
458    * FileOutputStream fos = new FileOutputStream("regions.dat");
459    * DataOutputStream dos = new DataOutputStream(fos);
460    * table.serializeRegionInfo(dos);
461    * dos.flush();
462    * dos.close();
463    * }
464    * </pre>
465    * @param out {@link DataOutput} to serialize this object into.
466    * @throws IOException if a remote or network exception occurs
467    */
468   public void serializeRegionInfo(DataOutput out) throws IOException {
469     Map<HRegionInfo, HServerAddress> allRegions = this.getRegionsInfo();
470     // first, write number of regions
471     out.writeInt(allRegions.size());
472     for (Map.Entry<HRegionInfo, HServerAddress> es : allRegions.entrySet()) {
473       es.getKey().write(out);
474       es.getValue().write(out);
475     }
476   }
477 
478   /**
479    * Read from <code>in</code> and deserialize the regions information.
480    *
481    * <p>It behaves similarly as {@link #getRegionsInfo getRegionsInfo}, except
482    * that it loads the region map from a {@link DataInput} object.
483    *
484    * <p>It is supposed to be followed immediately by  {@link
485    * #prewarmRegionCache prewarmRegionCache}.
486    *
487    * <p>
488    * Please refer to {@link #prewarmRegionCache prewarmRegionCache} for usage.
489    *
490    * @param in {@link DataInput} object.
491    * @return A map of HRegionInfo with its server address.
492    * @throws IOException if an I/O exception occurs.
493    */
494   public Map<HRegionInfo, HServerAddress> deserializeRegionInfo(DataInput in)
495   throws IOException {
496     final Map<HRegionInfo, HServerAddress> allRegions =
497       new TreeMap<HRegionInfo, HServerAddress>();
498 
499     // the first integer is expected to be the size of records
500     int regionsCount = in.readInt();
501     for (int i = 0; i < regionsCount; ++i) {
502       HRegionInfo hri = new HRegionInfo();
503       hri.readFields(in);
504       HServerAddress hsa = new HServerAddress();
505       hsa.readFields(in);
506       allRegions.put(hri, hsa);
507     }
508     return allRegions;
509   }
510 
511    @Override
512    public Result getRowOrBefore(final byte[] row, final byte[] family)
513    throws IOException {
514      return connection.getRegionServerWithRetries(
515          new ServerCallable<Result>(connection, tableName, row) {
516        public Result call() throws IOException {
517          return server.getClosestRowBefore(location.getRegionInfo().getRegionName(),
518            row, family);
519        }
520      });
521    }
522 
523   @Override
524   public ResultScanner getScanner(final Scan scan) throws IOException {
525     ClientScanner s = new ClientScanner(scan);
526     s.initialize();
527     return s;
528   }
529 
530   @Override
531   public ResultScanner getScanner(byte [] family) throws IOException {
532     Scan scan = new Scan();
533     scan.addFamily(family);
534     return getScanner(scan);
535   }
536 
537   @Override
538   public ResultScanner getScanner(byte [] family, byte [] qualifier)
539   throws IOException {
540     Scan scan = new Scan();
541     scan.addColumn(family, qualifier);
542     return getScanner(scan);
543   }
544 
545   public Result get(final Get get) throws IOException {
546     return connection.getRegionServerWithRetries(
547         new ServerCallable<Result>(connection, tableName, get.getRow()) {
548           public Result call() throws IOException {
549             return server.get(location.getRegionInfo().getRegionName(), get);
550           }
551         }
552     );
553   }
554 
555    public Result[] get(List<Get> gets) throws IOException {
556      try {
557        Object [] r1 = batch((List)gets);
558 
559        // translate.
560        Result [] results = new Result[r1.length];
561        int i=0;
562        for (Object o : r1) {
563          // batch ensures if there is a failure we get an exception instead
564          results[i++] = (Result) o;
565        }
566 
567        return results;
568      } catch (InterruptedException e) {
569        throw new IOException(e);
570      }
571    }
572 
573   /**
574    * Method that does a batch call on Deletes, Gets and Puts.  The ordering of
575    * execution of the actions is not defined. Meaning if you do a Put and a
576    * Get in the same {@link #batch} call, you will not necessarily be
577    * guaranteed that the Get returns what the Put had put.
578    *
579    * @param actions list of Get, Put, Delete objects
580    * @param results Empty Result[], same size as actions. Provides access to
581    * partial results, in case an exception is thrown. If there are any failures,
582    * there will be a null or Throwable will be in the results array, AND an
583    * exception will be thrown.
584    * @throws IOException
585    */
586   @Override
587   public synchronized void batch(final List<Row> actions, final Object[] results)
588       throws InterruptedException, IOException {
589     connection.processBatch(actions, tableName, pool, results);
590   }
591 
592   /**
593    * Method that does a batch call on Deletes, Gets and Puts.
594    *
595    * @param actions list of Get, Put, Delete objects
596    * @return the results from the actions. A null in the return array means that
597    * the call for that action failed, even after retries
598    * @throws IOException
599    */
600   @Override
601   public synchronized Object[] batch(final List<Row> actions) throws InterruptedException, IOException {
602     Object[] results = new Object[actions.size()];
603     connection.processBatch(actions, tableName, pool, results);
604     return results;
605   }
606 
607   /**
608    * Deletes the specified cells/row.
609    *
610    * @param delete The object that specifies what to delete.
611    * @throws IOException if a remote or network exception occurs.
612    * @since 0.20.0
613    */
614   @Override
615   public void delete(final Delete delete)
616   throws IOException {
617     connection.getRegionServerWithRetries(
618         new ServerCallable<Boolean>(connection, tableName, delete.getRow()) {
619           public Boolean call() throws IOException {
620             server.delete(location.getRegionInfo().getRegionName(), delete);
621             return null; // FindBugs NP_BOOLEAN_RETURN_NULL
622           }
623         }
624     );
625   }
626 
627   /**
628    * Deletes the specified cells/rows in bulk.
629    * @param deletes List of things to delete. As a side effect, it will be modified:
630    * successful {@link Delete}s are removed. The ordering of the list will not change.
631    * @throws IOException if a remote or network exception occurs. In that case
632    * the {@code deletes} argument will contain the {@link Delete} instances
633    * that have not be successfully applied.
634    * @since 0.20.1
635    * @see #batch(java.util.List, Object[])
636    */
637   @Override
638   public void delete(final List<Delete> deletes)
639   throws IOException {
640     Object[] results = new Object[deletes.size()];
641     try {
642       connection.processBatch((List) deletes, tableName, pool, results);
643     } catch (InterruptedException e) {
644       throw new IOException(e);
645     } finally {
646       // mutate list so that it is empty for complete success, or contains only failed records
647       // results are returned in the same order as the requests in list
648       // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
649       for (int i = results.length - 1; i>=0; i--) {
650         // if result is not null, it succeeded
651         if (results[i] instanceof Result) {
652           deletes.remove(i);
653         }
654       }
655     }
656   }
657 
658   @Override
659   public void put(final Put put) throws IOException {
660     doPut(Arrays.asList(put));
661   }
662 
663   @Override
664   public void put(final List<Put> puts) throws IOException {
665     doPut(puts);
666   }
667 
668   private void doPut(final List<Put> puts) throws IOException {
669     for (Put put : puts) {
670       validatePut(put);
671       writeBuffer.add(put);
672       currentWriteBufferSize += put.heapSize();
673     }
674     if (autoFlush || currentWriteBufferSize > writeBufferSize) {
675       flushCommits();
676     }
677   }
678 
679   @Override
680   public Result increment(final Increment increment) throws IOException {
681     if (!increment.hasFamilies()) {
682       throw new IOException(
683           "Invalid arguments to increment, no columns specified");
684     }
685     return connection.getRegionServerWithRetries(
686         new ServerCallable<Result>(connection, tableName, increment.getRow()) {
687           public Result call() throws IOException {
688             return server.increment(
689                 location.getRegionInfo().getRegionName(), increment);
690           }
691         }
692     );
693   }
694 
695   @Override
696   public long incrementColumnValue(final byte [] row, final byte [] family,
697       final byte [] qualifier, final long amount)
698   throws IOException {
699     return incrementColumnValue(row, family, qualifier, amount, true);
700   }
701 
702   @Override
703   public long incrementColumnValue(final byte [] row, final byte [] family,
704       final byte [] qualifier, final long amount, final boolean writeToWAL)
705   throws IOException {
706     NullPointerException npe = null;
707     if (row == null) {
708       npe = new NullPointerException("row is null");
709     } else if (family == null) {
710       npe = new NullPointerException("column is null");
711     }
712     if (npe != null) {
713       throw new IOException(
714           "Invalid arguments to incrementColumnValue", npe);
715     }
716     return connection.getRegionServerWithRetries(
717         new ServerCallable<Long>(connection, tableName, row) {
718           public Long call() throws IOException {
719             return server.incrementColumnValue(
720                 location.getRegionInfo().getRegionName(), row, family,
721                 qualifier, amount, writeToWAL);
722           }
723         }
724     );
725   }
726 
727   /**
728    * Atomically checks if a row/family/qualifier value match the expectedValue.
729    * If it does, it adds the put.  If value == null, checks for non-existence
730    * of the value.
731    *
732    * @param row to check
733    * @param family column family
734    * @param qualifier column qualifier
735    * @param value the expected value
736    * @param put put to execute if value matches.
737    * @throws IOException
738    * @return true if the new put was execute, false otherwise
739    */
740   @Override
741   public boolean checkAndPut(final byte [] row,
742       final byte [] family, final byte [] qualifier, final byte [] value,
743       final Put put)
744   throws IOException {
745     return connection.getRegionServerWithRetries(
746         new ServerCallable<Boolean>(connection, tableName, row) {
747           public Boolean call() throws IOException {
748             return server.checkAndPut(location.getRegionInfo().getRegionName(),
749                 row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE;
750           }
751         }
752     );
753   }
754 
755   /**
756    * Atomically checks if a row/family/qualifier value match the expectedValue.
757    * If it does, it adds the delete.  If value == null, checks for non-existence
758    * of the value.
759    *
760    * @param row to check
761    * @param family column family
762    * @param qualifier column qualifier
763    * @param value the expected value
764    * @param delete delete to execute if value matches.
765    * @throws IOException
766    * @return true if the new delete was executed, false otherwise
767    */
768   @Override
769   public boolean checkAndDelete(final byte [] row,
770       final byte [] family, final byte [] qualifier, final byte [] value,
771       final Delete delete)
772   throws IOException {
773     return connection.getRegionServerWithRetries(
774         new ServerCallable<Boolean>(connection, tableName, row) {
775           public Boolean call() throws IOException {
776             return server.checkAndDelete(
777                 location.getRegionInfo().getRegionName(),
778                 row, family, qualifier, value, delete)
779             ? Boolean.TRUE : Boolean.FALSE;
780           }
781         }
782     );
783   }
784 
785   /**
786    * Test for the existence of columns in the table, as specified in the Get.<p>
787    *
788    * This will return true if the Get matches one or more keys, false if not.<p>
789    *
790    * This is a server-side call so it prevents any data from being transfered
791    * to the client.
792    * @param get param to check for
793    * @return true if the specified Get matches one or more keys, false if not
794    * @throws IOException
795    */
796   @Override
797   public boolean exists(final Get get) throws IOException {
798     return connection.getRegionServerWithRetries(
799         new ServerCallable<Boolean>(connection, tableName, get.getRow()) {
800           public Boolean call() throws IOException {
801             return server.
802                 exists(location.getRegionInfo().getRegionName(), get);
803           }
804         }
805     );
806   }
807 
808   /**
809    * Executes all the buffered {@link Put} operations.
810    * <p>
811    * This method gets called once automatically for every {@link Put} or batch
812    * of {@link Put}s (when {@link #batch(List)} is used) when
813    * {@link #isAutoFlush()} is {@code true}.
814    * @throws IOException if a remote or network exception occurs.
815    */
816   @Override
817   public void flushCommits() throws IOException {
818     try {
819       connection.processBatchOfPuts(writeBuffer, tableName, pool);
820     } finally {
821       // the write buffer was adjusted by processBatchOfPuts
822       currentWriteBufferSize = 0;
823       for (Put aPut : writeBuffer) {
824         currentWriteBufferSize += aPut.heapSize();
825       }
826     }
827   }
828 
829   @Override
830   public void close() throws IOException {
831     flushCommits();
832   }
833 
834   // validate for well-formedness
835   private void validatePut(final Put put) throws IllegalArgumentException{
836     if (put.isEmpty()) {
837       throw new IllegalArgumentException("No columns to insert");
838     }
839     if (maxKeyValueSize > 0) {
840       for (List<KeyValue> list : put.getFamilyMap().values()) {
841         for (KeyValue kv : list) {
842           if (kv.getLength() > maxKeyValueSize) {
843             throw new IllegalArgumentException("KeyValue size too large");
844           }
845         }
846       }
847     }
848   }
849 
850   @Override
851   public RowLock lockRow(final byte [] row)
852   throws IOException {
853     return connection.getRegionServerWithRetries(
854       new ServerCallable<RowLock>(connection, tableName, row) {
855         public RowLock call() throws IOException {
856           long lockId =
857               server.lockRow(location.getRegionInfo().getRegionName(), row);
858           return new RowLock(row,lockId);
859         }
860       }
861     );
862   }
863 
864   @Override
865   public void unlockRow(final RowLock rl)
866   throws IOException {
867     connection.getRegionServerWithRetries(
868       new ServerCallable<Boolean>(connection, tableName, rl.getRow()) {
869         public Boolean call() throws IOException {
870           server.unlockRow(location.getRegionInfo().getRegionName(),
871               rl.getLockId());
872           return null; // FindBugs NP_BOOLEAN_RETURN_NULL
873         }
874       }
875     );
876   }
877 
878   @Override
879   public boolean isAutoFlush() {
880     return autoFlush;
881   }
882 
883   /**
884    * Turns 'auto-flush' on or off.
885    * <p>
886    * When enabled (default), {@link Put} operations don't get buffered/delayed
887    * and are immediately executed.  This is slower but safer.
888    * <p>
889    * Turning this off means that multiple {@link Put}s will be accepted before
890    * any RPC is actually sent to do the write operations.  If the application
891    * dies before pending writes get flushed to HBase, data will be lost.
892    * Other side effects may include the fact that the application thinks a
893    * {@link Put} was executed successfully whereas it was in fact only
894    * buffered and the operation may fail when attempting to flush all pending
895    * writes.  In that case though, the code will retry the failed {@link Put}
896    * upon its next attempt to flush the buffer.
897    *
898    * @param autoFlush Whether or not to enable 'auto-flush'.
899    * @see #flushCommits
900    */
901   public void setAutoFlush(boolean autoFlush) {
902     this.autoFlush = autoFlush;
903   }
904 
905   /**
906    * Returns the maximum size in bytes of the write buffer for this HTable.
907    * <p>
908    * The default value comes from the configuration parameter
909    * {@code hbase.client.write.buffer}.
910    * @return The size of the write buffer in bytes.
911    */
912   public long getWriteBufferSize() {
913     return writeBufferSize;
914   }
915 
916   /**
917    * Sets the size of the buffer in bytes.
918    * <p>
919    * If the new size is less than the current amount of data in the
920    * write buffer, the buffer gets flushed.
921    * @param writeBufferSize The new write buffer size, in bytes.
922    * @throws IOException if a remote or network exception occurs.
923    */
924   public void setWriteBufferSize(long writeBufferSize) throws IOException {
925     this.writeBufferSize = writeBufferSize;
926     if(currentWriteBufferSize > writeBufferSize) {
927       flushCommits();
928     }
929   }
930 
931   /**
932    * Returns the write buffer.
933    * @return The current write buffer.
934    */
935   public ArrayList<Put> getWriteBuffer() {
936     return writeBuffer;
937   }
938 
939   /**
940    * Implements the scanner interface for the HBase client.
941    * If there are multiple regions in a table, this scanner will iterate
942    * through them all.
943    */
944   protected class ClientScanner implements ResultScanner {
945     private final Log CLIENT_LOG = LogFactory.getLog(this.getClass());
946     // HEADSUP: The scan internal start row can change as we move through table.
947     private Scan scan;
948     private boolean closed = false;
949     // Current region scanner is against.  Gets cleared if current region goes
950     // wonky: e.g. if it splits on us.
951     private HRegionInfo currentRegion = null;
952     private ScannerCallable callable = null;
953     private final LinkedList<Result> cache = new LinkedList<Result>();
954     private final int caching;
955     private long lastNext;
956     // Keep lastResult returned successfully in case we have to reset scanner.
957     private Result lastResult = null;
958 
959     protected ClientScanner(final Scan scan) {
960       if (CLIENT_LOG.isDebugEnabled()) {
961         CLIENT_LOG.debug("Creating scanner over "
962             + Bytes.toString(getTableName())
963             + " starting at key '" + Bytes.toStringBinary(scan.getStartRow()) + "'");
964       }
965       this.scan = scan;
966       this.lastNext = System.currentTimeMillis();
967 
968       // Use the caching from the Scan.  If not set, use the default cache setting for this table.
969       if (this.scan.getCaching() > 0) {
970         this.caching = this.scan.getCaching();
971       } else {
972         this.caching = HTable.this.scannerCaching;
973       }
974 
975       // Removed filter validation.  We have a new format now, only one of all
976       // the current filters has a validate() method.  We can add it back,
977       // need to decide on what we're going to do re: filter redesign.
978       // Need, at the least, to break up family from qualifier as separate
979       // checks, I think it's important server-side filters are optimal in that
980       // respect.
981     }
982 
983     public void initialize() throws IOException {
984       nextScanner(this.caching, false);
985     }
986 
987     protected Scan getScan() {
988       return scan;
989     }
990 
991     protected long getTimestamp() {
992       return lastNext;
993     }
994 
995     // returns true if the passed region endKey
996     private boolean checkScanStopRow(final byte [] endKey) {
997       if (this.scan.getStopRow().length > 0) {
998         // there is a stop row, check to see if we are past it.
999         byte [] stopRow = scan.getStopRow();
1000         int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
1001           endKey, 0, endKey.length);
1002         if (cmp <= 0) {
1003           // stopRow <= endKey (endKey is equals to or larger than stopRow)
1004           // This is a stop.
1005           return true;
1006         }
1007       }
1008       return false; //unlikely.
1009     }
1010 
1011     /*
1012      * Gets a scanner for the next region.  If this.currentRegion != null, then
1013      * we will move to the endrow of this.currentRegion.  Else we will get
1014      * scanner at the scan.getStartRow().  We will go no further, just tidy
1015      * up outstanding scanners, if <code>currentRegion != null</code> and
1016      * <code>done</code> is true.
1017      * @param nbRows
1018      * @param done Server-side says we're done scanning.
1019      */
1020     private boolean nextScanner(int nbRows, final boolean done)
1021     throws IOException {
1022       // Close the previous scanner if it's open
1023       if (this.callable != null) {
1024         this.callable.setClose();
1025         getConnection().getRegionServerWithRetries(callable);
1026         this.callable = null;
1027       }
1028 
1029       // Where to start the next scanner
1030       byte [] localStartKey;
1031 
1032       // if we're at end of table, close and return false to stop iterating
1033       if (this.currentRegion != null) {
1034         byte [] endKey = this.currentRegion.getEndKey();
1035         if (endKey == null ||
1036             Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
1037             checkScanStopRow(endKey) ||
1038             done) {
1039           close();
1040           if (CLIENT_LOG.isDebugEnabled()) {
1041             CLIENT_LOG.debug("Finished with scanning at " + this.currentRegion);
1042           }
1043           return false;
1044         }
1045         localStartKey = endKey;
1046         if (CLIENT_LOG.isDebugEnabled()) {
1047           CLIENT_LOG.debug("Finished with region " + this.currentRegion);
1048         }
1049       } else {
1050         localStartKey = this.scan.getStartRow();
1051       }
1052 
1053       if (CLIENT_LOG.isDebugEnabled()) {
1054         CLIENT_LOG.debug("Advancing internal scanner to startKey at '" +
1055           Bytes.toStringBinary(localStartKey) + "'");
1056       }
1057       try {
1058         callable = getScannerCallable(localStartKey, nbRows);
1059         // Open a scanner on the region server starting at the
1060         // beginning of the region
1061         getConnection().getRegionServerWithRetries(callable);
1062         this.currentRegion = callable.getHRegionInfo();
1063       } catch (IOException e) {
1064         close();
1065         throw e;
1066       }
1067       return true;
1068     }
1069 
1070     protected ScannerCallable getScannerCallable(byte [] localStartKey,
1071         int nbRows) {
1072       scan.setStartRow(localStartKey);
1073       ScannerCallable s = new ScannerCallable(getConnection(),
1074         getTableName(), scan);
1075       s.setCaching(nbRows);
1076       return s;
1077     }
1078 
1079     public Result next() throws IOException {
1080       // If the scanner is closed but there is some rows left in the cache,
1081       // it will first empty it before returning null
1082       if (cache.size() == 0 && this.closed) {
1083         return null;
1084       }
1085       if (cache.size() == 0) {
1086         Result [] values = null;
1087         long remainingResultSize = maxScannerResultSize;
1088         int countdown = this.caching;
1089         // We need to reset it if it's a new callable that was created
1090         // with a countdown in nextScanner
1091         callable.setCaching(this.caching);
1092         // This flag is set when we want to skip the result returned.  We do
1093         // this when we reset scanner because it split under us.
1094         boolean skipFirst = false;
1095         do {
1096           try {
1097             // Server returns a null values if scanning is to stop.  Else,
1098             // returns an empty array if scanning is to go on and we've just
1099             // exhausted current region.
1100             values = getConnection().getRegionServerWithRetries(callable);
1101             if (skipFirst) {
1102               skipFirst = false;
1103               // Reget.
1104               values = getConnection().getRegionServerWithRetries(callable);
1105             }
1106           } catch (DoNotRetryIOException e) {
1107             if (e instanceof UnknownScannerException) {
1108               long timeout = lastNext + scannerTimeout;
1109               // If we are over the timeout, throw this exception to the client
1110               // Else, it's because the region moved and we used the old id
1111               // against the new region server; reset the scanner.
1112               if (timeout < System.currentTimeMillis()) {
1113                 long elapsed = System.currentTimeMillis() - lastNext;
1114                 ScannerTimeoutException ex = new ScannerTimeoutException(
1115                     elapsed + "ms passed since the last invocation, " +
1116                         "timeout is currently set to " + scannerTimeout);
1117                 ex.initCause(e);
1118                 throw ex;
1119               }
1120             } else {
1121               Throwable cause = e.getCause();
1122               if (cause == null || !(cause instanceof NotServingRegionException)) {
1123                 throw e;
1124               }
1125             }
1126             // Else, its signal from depths of ScannerCallable that we got an
1127             // NSRE on a next and that we need to reset the scanner.
1128             if (this.lastResult != null) {
1129               this.scan.setStartRow(this.lastResult.getRow());
1130               // Skip first row returned.  We already let it out on previous
1131               // invocation.
1132               skipFirst = true;
1133             }
1134             // Clear region
1135             this.currentRegion = null;
1136             continue;
1137           }
1138           lastNext = System.currentTimeMillis();
1139           if (values != null && values.length > 0) {
1140             for (Result rs : values) {
1141               cache.add(rs);
1142               for (KeyValue kv : rs.raw()) {
1143                   remainingResultSize -= kv.heapSize();
1144               }
1145               countdown--;
1146               this.lastResult = rs;
1147             }
1148           }
1149           // Values == null means server-side filter has determined we must STOP
1150         } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
1151       }
1152 
1153       if (cache.size() > 0) {
1154         return cache.poll();
1155       }
1156       return null;
1157     }
1158 
1159     /**
1160      * Get <param>nbRows</param> rows.
1161      * How many RPCs are made is determined by the {@link Scan#setCaching(int)}
1162      * setting (or hbase.client.scanner.caching in hbase-site.xml).
1163      * @param nbRows number of rows to return
1164      * @return Between zero and <param>nbRows</param> RowResults.  Scan is done
1165      * if returned array is of zero-length (We never return null).
1166      * @throws IOException
1167      */
1168     public Result [] next(int nbRows) throws IOException {
1169       // Collect values to be returned here
1170       ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
1171       for(int i = 0; i < nbRows; i++) {
1172         Result next = next();
1173         if (next != null) {
1174           resultSets.add(next);
1175         } else {
1176           break;
1177         }
1178       }
1179       return resultSets.toArray(new Result[resultSets.size()]);
1180     }
1181 
1182     public void close() {
1183       if (callable != null) {
1184         callable.setClose();
1185         try {
1186           getConnection().getRegionServerWithRetries(callable);
1187         } catch (IOException e) {
1188           // We used to catch this error, interpret, and rethrow. However, we
1189           // have since decided that it's not nice for a scanner's close to
1190           // throw exceptions. Chances are it was just an UnknownScanner
1191           // exception due to lease time out.
1192         }
1193         callable = null;
1194       }
1195       closed = true;
1196     }
1197 
1198     public Iterator<Result> iterator() {
1199       return new Iterator<Result>() {
1200         // The next RowResult, possibly pre-read
1201         Result next = null;
1202 
1203         // return true if there is another item pending, false if there isn't.
1204         // this method is where the actual advancing takes place, but you need
1205         // to call next() to consume it. hasNext() will only advance if there
1206         // isn't a pending next().
1207         public boolean hasNext() {
1208           if (next == null) {
1209             try {
1210               next = ClientScanner.this.next();
1211               return next != null;
1212             } catch (IOException e) {
1213               throw new RuntimeException(e);
1214             }
1215           }
1216           return true;
1217         }
1218 
1219         // get the pending next item and advance the iterator. returns null if
1220         // there is no next item.
1221         public Result next() {
1222           // since hasNext() does the real advancing, we call this to determine
1223           // if there is a next before proceeding.
1224           if (!hasNext()) {
1225             return null;
1226           }
1227 
1228           // if we get to here, then hasNext() has given us an item to return.
1229           // we want to return the item and then null out the next pointer, so
1230           // we use a temporary variable.
1231           Result temp = next;
1232           next = null;
1233           return temp;
1234         }
1235 
1236         public void remove() {
1237           throw new UnsupportedOperationException();
1238         }
1239       };
1240     }
1241   }
1242 
1243   static class DaemonThreadFactory implements ThreadFactory {
1244     static final AtomicInteger poolNumber = new AtomicInteger(1);
1245         final ThreadGroup group;
1246         final AtomicInteger threadNumber = new AtomicInteger(1);
1247         final String namePrefix;
1248 
1249         DaemonThreadFactory() {
1250             SecurityManager s = System.getSecurityManager();
1251             group = (s != null)? s.getThreadGroup() :
1252                                  Thread.currentThread().getThreadGroup();
1253             namePrefix = "pool-" +
1254                           poolNumber.getAndIncrement() +
1255                          "-thread-";
1256         }
1257 
1258         public Thread newThread(Runnable r) {
1259             Thread t = new Thread(group, r,
1260                                   namePrefix + threadNumber.getAndIncrement(),
1261                                   0);
1262             if (!t.isDaemon()) {
1263               t.setDaemon(true);
1264             }
1265             if (t.getPriority() != Thread.NORM_PRIORITY) {
1266               t.setPriority(Thread.NORM_PRIORITY);
1267             }
1268             return t;
1269         }
1270   }
1271 
1272   /**
1273    * Enable or disable region cache prefetch for the table. It will be
1274    * applied for the given table's all HTable instances who share the same
1275    * connection. By default, the cache prefetch is enabled.
1276    * @param tableName name of table to configure.
1277    * @param enable Set to true to enable region cache prefetch. Or set to
1278    * false to disable it.
1279    * @throws ZooKeeperConnectionException
1280    */
1281   public static void setRegionCachePrefetch(final byte[] tableName,
1282       boolean enable) throws ZooKeeperConnectionException {
1283     HConnectionManager.getConnection(HBaseConfiguration.create()).
1284     setRegionCachePrefetch(tableName, enable);
1285   }
1286 
1287   /**
1288    * Enable or disable region cache prefetch for the table. It will be
1289    * applied for the given table's all HTable instances who share the same
1290    * connection. By default, the cache prefetch is enabled.
1291    * @param conf The Configuration object to use.
1292    * @param tableName name of table to configure.
1293    * @param enable Set to true to enable region cache prefetch. Or set to
1294    * false to disable it.
1295    * @throws ZooKeeperConnectionException
1296    */
1297   public static void setRegionCachePrefetch(final Configuration conf,
1298       final byte[] tableName, boolean enable) throws ZooKeeperConnectionException {
1299     HConnectionManager.getConnection(conf).setRegionCachePrefetch(
1300         tableName, enable);
1301   }
1302 
1303   /**
1304    * Check whether region cache prefetch is enabled or not for the table.
1305    * @param conf The Configuration object to use.
1306    * @param tableName name of table to check
1307    * @return true if table's region cache prefecth is enabled. Otherwise
1308    * it is disabled.
1309    * @throws ZooKeeperConnectionException
1310    */
1311   public static boolean getRegionCachePrefetch(final Configuration conf,
1312       final byte[] tableName) throws ZooKeeperConnectionException {
1313     return HConnectionManager.getConnection(conf).getRegionCachePrefetch(
1314         tableName);
1315   }
1316 
1317   /**
1318    * Check whether region cache prefetch is enabled or not for the table.
1319    * @param tableName name of table to check
1320    * @return true if table's region cache prefecth is enabled. Otherwise
1321    * it is disabled.
1322    * @throws ZooKeeperConnectionException
1323    */
1324   public static boolean getRegionCachePrefetch(final byte[] tableName) throws ZooKeeperConnectionException {
1325     return HConnectionManager.getConnection(HBaseConfiguration.create()).
1326     getRegionCachePrefetch(tableName);
1327   }
1328 }