View Javadoc

1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.DataInput;
24  import java.io.DataOutput;
25  import java.io.IOException;
26  import java.lang.reflect.Proxy;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Collections;
31  import java.util.NavigableMap;
32  import java.util.TreeMap;
33  import java.util.concurrent.ExecutorService;
34  import java.util.concurrent.SynchronousQueue;
35  import java.util.concurrent.ThreadPoolExecutor;
36  import java.util.concurrent.TimeUnit;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.HRegionLocation;
45  import org.apache.hadoop.hbase.HServerAddress;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.KeyValue;
48  import org.apache.hadoop.hbase.ServerName;
49  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
50  import org.apache.hadoop.hbase.client.coprocessor.Batch;
51  import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
52  import org.apache.hadoop.hbase.ipc.ExecRPCInvoker;
53  import org.apache.hadoop.hbase.util.Addressing;
54  import org.apache.hadoop.hbase.util.Bytes;
55  import org.apache.hadoop.hbase.util.Pair;
56  import org.apache.hadoop.hbase.util.Threads;
57  
58  /**
59   * <p>Used to communicate with a single HBase table.
60   *
61   * <p>This class is not thread safe for reads nor write.
62   * 
63   * <p>In case of writes (Put, Delete), the underlying write buffer can
64   * be corrupted if multiple threads contend over a single HTable instance.
65   * 
66   * <p>In case of reads, some fields used by a Scan are shared among all threads.
67   * The HTable implementation can either not contract to be safe in case of a Get
68   *
69   * <p>To access a table in a multi threaded environment, please consider
70   * using the {@link HTablePool} class to create your HTable instances.
71   *
72   * <p>Instances of HTable passed the same {@link Configuration} instance will
73   * share connections to servers out on the cluster and to the zookeeper ensemble
74   * as well as caches of region locations.  This is usually a *good* thing and it
75   * is recommended to reuse the same configuration object for all your tables.
76   * This happens because they will all share the same underlying
77   * {@link HConnection} instance. See {@link HConnectionManager} for more on
78   * how this mechanism works.
79   *
80   * <p>{@link HConnection} will read most of the
81   * configuration it needs from the passed {@link Configuration} on initial
82   * construction.  Thereafter, for settings such as
83   * <code>hbase.client.pause</code>, <code>hbase.client.retries.number</code>,
84   * and <code>hbase.client.rpc.maxattempts</code> updating their values in the
85   * passed {@link Configuration} subsequent to {@link HConnection} construction
86   * will go unnoticed.  To run with changed values, make a new
87   * {@link HTable} passing a new {@link Configuration} instance that has the
88   * new configuration.
89   *
90   * <p>Note that this class implements the {@link Closeable} interface. When a
91   * HTable instance is no longer required, it *should* be closed in order to ensure
92   * that the underlying resources are promptly released. Please note that the close 
93   * method can throw java.io.IOException that must be handled.
94   *
95   * @see HBaseAdmin for create, drop, list, enable and disable of tables.
96   * @see HConnection
97   * @see HConnectionManager
98   */
99  public class HTable implements HTableInterface {
100   private static final Log LOG = LogFactory.getLog(HTable.class);
101   private HConnection connection;
102   private final byte [] tableName;
103   private volatile Configuration configuration;
104   private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
105   private long writeBufferSize;
106   private boolean clearBufferOnFail;
107   private boolean autoFlush;
108   private long currentWriteBufferSize;
109   protected int scannerCaching;
110   private int maxKeyValueSize;
111   private ExecutorService pool;  // For Multi
112   private boolean closed;
113   private int operationTimeout;
114   private final boolean cleanupPoolOnClose; // shutdown the pool in close()
115   private final boolean cleanupConnectionOnClose; // close the connection in close()
116 
117   /**
118    * Creates an object to access a HBase table.
119    * Shares zookeeper connection and other resources with other HTable instances
120    * created with the same <code>conf</code> instance.  Uses already-populated
121    * region cache if one is available, populated by any other HTable instances
122    * sharing this <code>conf</code> instance.  Recommended.
123    * @param conf Configuration object to use.
124    * @param tableName Name of the table.
125    * @throws IOException if a remote or network exception occurs
126    */
127   public HTable(Configuration conf, final String tableName)
128   throws IOException {
129     this(conf, Bytes.toBytes(tableName));
130   }
131 
132 
133   /**
134    * Creates an object to access a HBase table.
135    * Shares zookeeper connection and other resources with other HTable instances
136    * created with the same <code>conf</code> instance.  Uses already-populated
137    * region cache if one is available, populated by any other HTable instances
138    * sharing this <code>conf</code> instance.  Recommended.
139    * @param conf Configuration object to use.
140    * @param tableName Name of the table.
141    * @throws IOException if a remote or network exception occurs
142    */
143   public HTable(Configuration conf, final byte [] tableName)
144   throws IOException {
145     this.tableName = tableName;
146     this.cleanupPoolOnClose = this.cleanupConnectionOnClose = true;
147     if (conf == null) {
148       this.connection = null;
149       return;
150     }
151     this.connection = HConnectionManager.getConnection(conf);
152     this.configuration = conf;
153 
154     int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE);
155     if (maxThreads == 0) {
156       maxThreads = 1; // is there a better default?
157     }
158     long keepAliveTime = conf.getLong("hbase.htable.threads.keepalivetime", 60);
159 
160     // Using the "direct handoff" approach, new threads will only be created
161     // if it is necessary and will grow unbounded. This could be bad but in HCM
162     // we only create as many Runnables as there are region servers. It means
163     // it also scales when new region servers are added.
164     this.pool = new ThreadPoolExecutor(1, maxThreads,
165         keepAliveTime, TimeUnit.SECONDS,
166         new SynchronousQueue<Runnable>(),
167         Threads.newDaemonThreadFactory("hbase-table"));
168     ((ThreadPoolExecutor)this.pool).allowCoreThreadTimeOut(true);
169 
170     this.finishSetup();
171   }
172 
173   /**
174    * Creates an object to access a HBase table.
175    * Shares zookeeper connection and other resources with other HTable instances
176    * created with the same <code>conf</code> instance.  Uses already-populated
177    * region cache if one is available, populated by any other HTable instances
178    * sharing this <code>conf</code> instance.
179    * Use this constructor when the ExecutorService is externally managed.
180    * @param conf Configuration object to use.
181    * @param tableName Name of the table.
182    * @param pool ExecutorService to be used.
183    * @throws IOException if a remote or network exception occurs
184    */
185   public HTable(Configuration conf, final byte[] tableName, final ExecutorService pool)
186       throws IOException {
187     this.connection = HConnectionManager.getConnection(conf);
188     this.configuration = conf;
189     this.pool = pool;
190     this.tableName = tableName;
191     this.cleanupPoolOnClose = false;
192     this.cleanupConnectionOnClose = true;
193 
194     this.finishSetup();
195   }
196 
197   /**
198    * Creates an object to access a HBase table.
199    * Shares zookeeper connection and other resources with other HTable instances
200    * created with the same <code>connection</code> instance.
201    * Use this constructor when the ExecutorService and HConnection instance are
202    * externally managed.
203    * @param tableName Name of the table.
204    * @param connection HConnection to be used.
205    * @param pool ExecutorService to be used.
206    * @throws IOException if a remote or network exception occurs
207    */
208   public HTable(final byte[] tableName, final HConnection connection, 
209       final ExecutorService pool) throws IOException {
210     if (connection == null || connection.isClosed()) {
211       throw new IllegalArgumentException("Connection is null or closed.");
212     }
213     this.tableName = tableName;
214     this.cleanupPoolOnClose = this.cleanupConnectionOnClose = false;
215     this.connection = connection;
216     this.configuration = connection.getConfiguration();
217     this.pool = pool;
218 
219     this.finishSetup();
220   }
221 
222   /**
223    * setup this HTable's parameter based on the passed configuration
224    * @param conf
225    */
226   private void finishSetup() throws IOException {
227     this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
228     this.operationTimeout = HTableDescriptor.isMetaTable(tableName) ? HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT
229         : this.configuration.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
230             HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
231     this.writeBufferSize = this.configuration.getLong(
232         "hbase.client.write.buffer", 2097152);
233     this.clearBufferOnFail = true;
234     this.autoFlush = true;
235     this.currentWriteBufferSize = 0;
236     this.scannerCaching = this.configuration.getInt(
237         "hbase.client.scanner.caching", 1);
238 
239     this.maxKeyValueSize = this.configuration.getInt(
240         "hbase.client.keyvalue.maxsize", -1);
241     this.closed = false;
242   }
243 
244   /**
245    * {@inheritDoc}
246    */
247   @Override
248   public Configuration getConfiguration() {
249     return configuration;
250   }
251 
252   /**
253    * Tells whether or not a table is enabled or not. This method creates a
254    * new HBase configuration, so it might make your unit tests fail due to
255    * incorrect ZK client port.
256    * @param tableName Name of table to check.
257    * @return {@code true} if table is online.
258    * @throws IOException if a remote or network exception occurs
259 	* @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
260    */
261   @Deprecated
262   public static boolean isTableEnabled(String tableName) throws IOException {
263     return isTableEnabled(Bytes.toBytes(tableName));
264   }
265 
266   /**
267    * Tells whether or not a table is enabled or not. This method creates a
268    * new HBase configuration, so it might make your unit tests fail due to
269    * incorrect ZK client port.
270    * @param tableName Name of table to check.
271    * @return {@code true} if table is online.
272    * @throws IOException if a remote or network exception occurs
273    * @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
274    */
275   @Deprecated
276   public static boolean isTableEnabled(byte[] tableName) throws IOException {
277     return isTableEnabled(HBaseConfiguration.create(), tableName);
278   }
279 
280   /**
281    * Tells whether or not a table is enabled or not.
282    * @param conf The Configuration object to use.
283    * @param tableName Name of table to check.
284    * @return {@code true} if table is online.
285    * @throws IOException if a remote or network exception occurs
286 	* @deprecated use {@link HBaseAdmin#isTableEnabled(byte[])}
287    */
288   @Deprecated
289   public static boolean isTableEnabled(Configuration conf, String tableName)
290   throws IOException {
291     return isTableEnabled(conf, Bytes.toBytes(tableName));
292   }
293 
294   /**
295    * Tells whether or not a table is enabled or not.
296    * @param conf The Configuration object to use.
297    * @param tableName Name of table to check.
298    * @return {@code true} if table is online.
299    * @throws IOException if a remote or network exception occurs
300    */
301   public static boolean isTableEnabled(Configuration conf,
302       final byte[] tableName) throws IOException {
303     return HConnectionManager.execute(new HConnectable<Boolean>(conf) {
304       @Override
305       public Boolean connect(HConnection connection) throws IOException {
306         return connection.isTableEnabled(tableName);
307       }
308     });
309   }
310 
311   /**
312    * Find region location hosting passed row using cached info
313    * @param row Row to find.
314    * @return The location of the given row.
315    * @throws IOException if a remote or network exception occurs
316    */
317   public HRegionLocation getRegionLocation(final String row)
318   throws IOException {
319     return connection.getRegionLocation(tableName, Bytes.toBytes(row), false);
320   }
321 
322   /**
323    * Finds the region on which the given row is being served.
324    * @param row Row to find.
325    * @return Location of the row.
326    * @throws IOException if a remote or network exception occurs
327    * @deprecated use {@link #getRegionLocation(byte [], boolean)} instead
328    */
329   public HRegionLocation getRegionLocation(final byte [] row)
330   throws IOException {
331     return connection.getRegionLocation(tableName, row, false);
332   }
333 
334   /**
335    * Finds the region on which the given row is being served.
336    * @param row Row to find.
337    * @param reload whether or not to reload information or just use cached
338    * information
339    * @return Location of the row.
340    * @throws IOException if a remote or network exception occurs
341    */
342   public HRegionLocation getRegionLocation(final byte [] row, boolean reload)
343   throws IOException {
344     return connection.getRegionLocation(tableName, row, reload);
345   }
346      
347   /**
348    * {@inheritDoc}
349    */
350   @Override
351   public byte [] getTableName() {
352     return this.tableName;
353   }
354 
355   /**
356    * <em>INTERNAL</em> Used by unit tests and tools to do low-level
357    * manipulations.
358    * @return An HConnection instance.
359    * @deprecated This method will be changed from public to package protected.
360    */
361   // TODO(tsuna): Remove this.  Unit tests shouldn't require public helpers.
362   public HConnection getConnection() {
363     return this.connection;
364   }
365 
366   /**
367    * Gets the number of rows that a scanner will fetch at once.
368    * <p>
369    * The default value comes from {@code hbase.client.scanner.caching}.
370    * @deprecated Use {@link Scan#setCaching(int)} and {@link Scan#getCaching()}
371    */
372   public int getScannerCaching() {
373     return scannerCaching;
374   }
375 
376   /**
377    * Sets the number of rows that a scanner will fetch at once.
378    * <p>
379    * This will override the value specified by
380    * {@code hbase.client.scanner.caching}.
381    * Increasing this value will reduce the amount of work needed each time
382    * {@code next()} is called on a scanner, at the expense of memory use
383    * (since more rows will need to be maintained in memory by the scanners).
384    * @param scannerCaching the number of rows a scanner will fetch at once.
385    * @deprecated Use {@link Scan#setCaching(int)}
386    */
387   public void setScannerCaching(int scannerCaching) {
388     this.scannerCaching = scannerCaching;
389   }
390 
391   /**
392    * {@inheritDoc}
393    */
394   @Override
395   public HTableDescriptor getTableDescriptor() throws IOException {
396     return new UnmodifyableHTableDescriptor(
397       this.connection.getHTableDescriptor(this.tableName));
398   }
399 
400   /**
401    * Gets the starting row key for every region in the currently open table.
402    * <p>
403    * This is mainly useful for the MapReduce integration.
404    * @return Array of region starting row keys
405    * @throws IOException if a remote or network exception occurs
406    */
407   public byte [][] getStartKeys() throws IOException {
408     return getStartEndKeys().getFirst();
409   }
410 
411   /**
412    * Gets the ending row key for every region in the currently open table.
413    * <p>
414    * This is mainly useful for the MapReduce integration.
415    * @return Array of region ending row keys
416    * @throws IOException if a remote or network exception occurs
417    */
418   public byte[][] getEndKeys() throws IOException {
419     return getStartEndKeys().getSecond();
420   }
421 
422   /**
423    * Gets the starting and ending row keys for every region in the currently
424    * open table.
425    * <p>
426    * This is mainly useful for the MapReduce integration.
427    * @return Pair of arrays of region starting and ending row keys
428    * @throws IOException if a remote or network exception occurs
429    */
430   public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException {
431     NavigableMap<HRegionInfo, ServerName> regions = getRegionLocations();
432     final List<byte[]> startKeyList = new ArrayList<byte[]>(regions.size());
433     final List<byte[]> endKeyList = new ArrayList<byte[]>(regions.size());
434 
435     for (HRegionInfo region : regions.keySet()) {
436       startKeyList.add(region.getStartKey());
437       endKeyList.add(region.getEndKey());
438     }
439 
440     return new Pair<byte [][], byte [][]>(
441       startKeyList.toArray(new byte[startKeyList.size()][]),
442       endKeyList.toArray(new byte[endKeyList.size()][]));
443   }
444 
445   /**
446    * Gets all the regions and their address for this table.
447    * @return A map of HRegionInfo with it's server address
448    * @throws IOException if a remote or network exception occurs
449    * @deprecated Use {@link #getRegionLocations()} or {@link #getStartEndKeys()}
450    */
451   public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException {
452     final Map<HRegionInfo, HServerAddress> regionMap =
453       new TreeMap<HRegionInfo, HServerAddress>();
454 
455     final Map<HRegionInfo, ServerName> regionLocations = getRegionLocations();
456 
457     for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) {
458       HServerAddress server = new HServerAddress();
459       ServerName serverName = entry.getValue();
460       if (serverName != null && serverName.getHostAndPort() != null) {
461         server = new HServerAddress(Addressing.createInetSocketAddressFromHostAndPortStr(
462             serverName.getHostAndPort()));
463       }
464       regionMap.put(entry.getKey(), server);
465     }
466 
467     return regionMap;
468   }
469 
470   /**
471    * Gets all the regions and their address for this table.
472    * <p>
473    * This is mainly useful for the MapReduce integration.
474    * @return A map of HRegionInfo with it's server address
475    * @throws IOException if a remote or network exception occurs
476    */
477   public NavigableMap<HRegionInfo, ServerName> getRegionLocations() throws IOException {
478     return MetaScanner.allTableRegions(getConfiguration(), this.connection, getTableName(), false);
479   }
480 
481   /**
482    * Get the corresponding regions for an arbitrary range of keys.
483    * <p>
484    * @param startRow Starting row in range, inclusive
485    * @param endRow Ending row in range, exclusive
486    * @return A list of HRegionLocations corresponding to the regions that
487    * contain the specified range
488    * @throws IOException if a remote or network exception occurs
489    */
490   public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
491     final byte [] endKey) throws IOException {
492     return getRegionsInRange(startKey, endKey, false);
493   }
494 
495   /**
496    * Get the corresponding regions for an arbitrary range of keys.
497    * <p>
498    * @param startKey Starting row in range, inclusive
499    * @param endKey Ending row in range, exclusive
500    * @param reload true to reload information or false to use cached information
501    * @return A list of HRegionLocations corresponding to the regions that
502    * contain the specified range
503    * @throws IOException if a remote or network exception occurs
504    */
505   public List<HRegionLocation> getRegionsInRange(final byte [] startKey,
506       final byte [] endKey, final boolean reload) throws IOException {
507     return getKeysAndRegionsInRange(startKey, endKey, false, reload).getSecond();
508   }
509 
510   /**
511    * Get the corresponding start keys and regions for an arbitrary range of
512    * keys.
513    * <p>
514    * @param startKey Starting row in range, inclusive
515    * @param endKey Ending row in range
516    * @param includeEndKey true if endRow is inclusive, false if exclusive
517    * @return A pair of list of start keys and list of HRegionLocations that
518    *         contain the specified range
519    * @throws IOException if a remote or network exception occurs
520    */
521   private Pair<List<byte[]>, List<HRegionLocation>> getKeysAndRegionsInRange(
522       final byte[] startKey, final byte[] endKey, final boolean includeEndKey)
523       throws IOException {
524     return getKeysAndRegionsInRange(startKey, endKey, includeEndKey, false);
525   }
526 
527   /**
528    * Get the corresponding start keys and regions for an arbitrary range of
529    * keys.
530    * <p>
531    * @param startKey Starting row in range, inclusive
532    * @param endKey Ending row in range
533    * @param includeEndKey true if endRow is inclusive, false if exclusive
534    * @param reload true to reload information or false to use cached information
535    * @return A pair of list of start keys and list of HRegionLocations that
536    *         contain the specified range
537    * @throws IOException if a remote or network exception occurs
538    */
539   private Pair<List<byte[]>, List<HRegionLocation>> getKeysAndRegionsInRange(
540       final byte[] startKey, final byte[] endKey, final boolean includeEndKey,
541       final boolean reload) throws IOException {
542     final boolean endKeyIsEndOfTable = Bytes.equals(endKey,HConstants.EMPTY_END_ROW);
543     if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) {
544       throw new IllegalArgumentException(
545         "Invalid range: " + Bytes.toStringBinary(startKey) +
546         " > " + Bytes.toStringBinary(endKey));
547     }
548     List<byte[]> keysInRange = new ArrayList<byte[]>();
549     List<HRegionLocation> regionsInRange = new ArrayList<HRegionLocation>();
550     byte[] currentKey = startKey;
551     do {
552       HRegionLocation regionLocation = getRegionLocation(currentKey, reload);
553       keysInRange.add(currentKey);
554       regionsInRange.add(regionLocation);
555       currentKey = regionLocation.getRegionInfo().getEndKey();
556     } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
557         && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0
558             || (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0)));
559     return new Pair<List<byte[]>, List<HRegionLocation>>(keysInRange,
560         regionsInRange);
561   }
562 
563   /**
564    * Save the passed region information and the table's regions
565    * cache.
566    * <p>
567    * This is mainly useful for the MapReduce integration. You can call
568    * {@link #deserializeRegionInfo deserializeRegionInfo}
569    * to deserialize regions information from a
570    * {@link DataInput}, then call this method to load them to cache.
571    *
572    * <pre>
573    * {@code
574    * HTable t1 = new HTable("foo");
575    * FileInputStream fis = new FileInputStream("regions.dat");
576    * DataInputStream dis = new DataInputStream(fis);
577    *
578    * Map<HRegionInfo, HServerAddress> hm = t1.deserializeRegionInfo(dis);
579    * t1.prewarmRegionCache(hm);
580    * }
581    * </pre>
582    * @param regionMap This piece of regions information will be loaded
583    * to region cache.
584    */
585   public void prewarmRegionCache(Map<HRegionInfo, HServerAddress> regionMap) {
586     this.connection.prewarmRegionCache(this.getTableName(), regionMap);
587   }
588 
589   /**
590    * Serialize the regions information of this table and output
591    * to <code>out</code>.
592    * <p>
593    * This is mainly useful for the MapReduce integration. A client could
594    * perform a large scan for all the regions for the table, serialize the
595    * region info to a file. MR job can ship a copy of the meta for the table in
596    * the DistributedCache.
597    * <pre>
598    * {@code
599    * FileOutputStream fos = new FileOutputStream("regions.dat");
600    * DataOutputStream dos = new DataOutputStream(fos);
601    * table.serializeRegionInfo(dos);
602    * dos.flush();
603    * dos.close();
604    * }
605    * </pre>
606    * @param out {@link DataOutput} to serialize this object into.
607    * @throws IOException if a remote or network exception occurs
608    */
609   public void serializeRegionInfo(DataOutput out) throws IOException {
610     Map<HRegionInfo, HServerAddress> allRegions = this.getRegionsInfo();
611     // first, write number of regions
612     out.writeInt(allRegions.size());
613     for (Map.Entry<HRegionInfo, HServerAddress> es : allRegions.entrySet()) {
614       es.getKey().write(out);
615       es.getValue().write(out);
616     }
617   }
618 
619   /**
620    * Read from <code>in</code> and deserialize the regions information.
621    *
622    * <p>It behaves similarly as {@link #getRegionsInfo getRegionsInfo}, except
623    * that it loads the region map from a {@link DataInput} object.
624    *
625    * <p>It is supposed to be followed immediately by  {@link
626    * #prewarmRegionCache prewarmRegionCache}.
627    *
628    * <p>
629    * Please refer to {@link #prewarmRegionCache prewarmRegionCache} for usage.
630    *
631    * @param in {@link DataInput} object.
632    * @return A map of HRegionInfo with its server address.
633    * @throws IOException if an I/O exception occurs.
634    */
635   public Map<HRegionInfo, HServerAddress> deserializeRegionInfo(DataInput in)
636   throws IOException {
637     final Map<HRegionInfo, HServerAddress> allRegions =
638       new TreeMap<HRegionInfo, HServerAddress>();
639 
640     // the first integer is expected to be the size of records
641     int regionsCount = in.readInt();
642     for (int i = 0; i < regionsCount; ++i) {
643       HRegionInfo hri = new HRegionInfo();
644       hri.readFields(in);
645       HServerAddress hsa = new HServerAddress();
646       hsa.readFields(in);
647       allRegions.put(hri, hsa);
648     }
649     return allRegions;
650   }
651 
652   /**
653    * {@inheritDoc}
654    */
655    @Override
656    public Result getRowOrBefore(final byte[] row, final byte[] family)
657    throws IOException {
658      return new ServerCallable<Result>(connection, tableName, row, operationTimeout) {
659        public Result call() throws IOException {
660          return server.getClosestRowBefore(location.getRegionInfo().getRegionName(),
661            row, family);
662        }
663      }.withRetries();
664    }
665 
666    /**
667     * {@inheritDoc}
668     */
669   @Override
670   public ResultScanner getScanner(final Scan scan) throws IOException {
671     if (scan.getCaching() <= 0) {
672       scan.setCaching(getScannerCaching());
673     }
674     return new ClientScanner(getConfiguration(), scan, getTableName(),
675         this.connection);
676   }
677 
678   /**
679    * {@inheritDoc}
680    */
681   @Override
682   public ResultScanner getScanner(byte [] family) throws IOException {
683     Scan scan = new Scan();
684     scan.addFamily(family);
685     return getScanner(scan);
686   }
687 
688   /**
689    * {@inheritDoc}
690    */
691   @Override
692   public ResultScanner getScanner(byte [] family, byte [] qualifier)
693   throws IOException {
694     Scan scan = new Scan();
695     scan.addColumn(family, qualifier);
696     return getScanner(scan);
697   }
698 
699   /**
700    * {@inheritDoc}
701    */
702   @Override
703   public Result get(final Get get) throws IOException {
704     return new ServerCallable<Result>(connection, tableName, get.getRow(), operationTimeout) {
705           public Result call() throws IOException {
706             return server.get(location.getRegionInfo().getRegionName(), get);
707           }
708         }.withRetries();
709   }
710 
711   /**
712    * {@inheritDoc}
713    */
714   @Override
715   public Result[] get(List<Get> gets) throws IOException {
716     if (gets.size() == 1) {
717       return new Result[]{get(gets.get(0))};
718     }
719     try {
720       Object [] r1 = batch((List)gets);
721 
722       // translate.
723       Result [] results = new Result[r1.length];
724       int i=0;
725       for (Object o : r1) {
726         // batch ensures if there is a failure we get an exception instead
727         results[i++] = (Result) o;
728       }
729 
730       return results;
731     } catch (InterruptedException e) {
732       throw new IOException(e);
733     }
734   }
735 
736   /**
737    * {@inheritDoc}
738    */
739   @Override
740   public void batch(final List<?extends Row> actions, final Object[] results)
741       throws InterruptedException, IOException {
742     connection.processBatch(actions, tableName, pool, results);
743   }
744 
745   /**
746    * {@inheritDoc}
747    */
748   @Override
749   public Object[] batch(final List<? extends Row> actions) throws InterruptedException, IOException {
750     Object[] results = new Object[actions.size()];
751     connection.processBatch(actions, tableName, pool, results);
752     return results;
753   }
754 
755   /**
756    * {@inheritDoc}
757    */
758   @Override
759   public void delete(final Delete delete)
760   throws IOException {
761     new ServerCallable<Boolean>(connection, tableName, delete.getRow(), operationTimeout) {
762           public Boolean call() throws IOException {
763             server.delete(location.getRegionInfo().getRegionName(), delete);
764             return null; // FindBugs NP_BOOLEAN_RETURN_NULL
765           }
766         }.withRetries();
767   }
768 
769   /**
770    * {@inheritDoc}
771    */
772   @Override
773   public void delete(final List<Delete> deletes)
774   throws IOException {
775     Object[] results = new Object[deletes.size()];
776     try {
777       connection.processBatch((List) deletes, tableName, pool, results);
778     } catch (InterruptedException e) {
779       throw new IOException(e);
780     } finally {
781       // mutate list so that it is empty for complete success, or contains only failed records
782       // results are returned in the same order as the requests in list
783       // walk the list backwards, so we can remove from list without impacting the indexes of earlier members
784       for (int i = results.length - 1; i>=0; i--) {
785         // if result is not null, it succeeded
786         if (results[i] instanceof Result) {
787           deletes.remove(i);
788         }
789       }
790     }
791   }
792 
793   /**
794    * {@inheritDoc}
795    */
796   @Override
797   public void put(final Put put) throws IOException {
798     doPut(put);
799     if (autoFlush) {
800       flushCommits();
801     }
802   }
803 
804   /**
805    * {@inheritDoc}
806    */
807   @Override
808   public void put(final List<Put> puts) throws IOException {
809     for (Put put : puts) {
810       doPut(put);
811     }
812     if (autoFlush) {
813       flushCommits();
814     }
815   }
816 
817   private void doPut(Put put) throws IOException{
818     validatePut(put);
819     writeBuffer.add(put);
820     currentWriteBufferSize += put.heapSize();
821     if (currentWriteBufferSize > writeBufferSize) {
822       flushCommits();
823     }
824   }
825 
826   /**
827    * {@inheritDoc}
828    */
829   @Override
830   public void mutateRow(final RowMutations rm) throws IOException {
831     new ServerCallable<Void>(connection, tableName, rm.getRow(),
832         operationTimeout) {
833       public Void call() throws IOException {
834         server.mutateRow(location.getRegionInfo().getRegionName(), rm);
835         return null;
836       }
837     }.withRetries();
838   }
839 
840   /**
841    * {@inheritDoc}
842    */
843   @Override
844   public Result append(final Append append) throws IOException {
845     if (append.numFamilies() == 0) {
846       throw new IOException(
847           "Invalid arguments to append, no columns specified");
848     }
849     return new ServerCallable<Result>(connection, tableName, append.getRow(), operationTimeout) {
850           public Result call() throws IOException {
851             return server.append(
852                 location.getRegionInfo().getRegionName(), append);
853           }
854         }.withRetries();
855   }
856 
857   /**
858    * {@inheritDoc}
859    */
860   @Override
861   public Result increment(final Increment increment) throws IOException {
862     if (!increment.hasFamilies()) {
863       throw new IOException(
864           "Invalid arguments to increment, no columns specified");
865     }
866     return new ServerCallable<Result>(connection, tableName, increment.getRow(), operationTimeout) {
867           public Result call() throws IOException {
868             return server.increment(
869                 location.getRegionInfo().getRegionName(), increment);
870           }
871         }.withRetries();
872   }
873 
874   /**
875    * {@inheritDoc}
876    */
877   @Override
878   public long incrementColumnValue(final byte [] row, final byte [] family,
879       final byte [] qualifier, final long amount)
880   throws IOException {
881     return incrementColumnValue(row, family, qualifier, amount, true);
882   }
883 
884   /**
885    * {@inheritDoc}
886    */
887   @Override
888   public long incrementColumnValue(final byte [] row, final byte [] family,
889       final byte [] qualifier, final long amount, final boolean writeToWAL)
890   throws IOException {
891     NullPointerException npe = null;
892     if (row == null) {
893       npe = new NullPointerException("row is null");
894     } else if (family == null) {
895       npe = new NullPointerException("column is null");
896     }
897     if (npe != null) {
898       throw new IOException(
899           "Invalid arguments to incrementColumnValue", npe);
900     }
901     return new ServerCallable<Long>(connection, tableName, row, operationTimeout) {
902           public Long call() throws IOException {
903             return server.incrementColumnValue(
904                 location.getRegionInfo().getRegionName(), row, family,
905                 qualifier, amount, writeToWAL);
906           }
907         }.withRetries();
908   }
909 
910   /**
911    * {@inheritDoc}
912    */
913   @Override
914   public boolean checkAndPut(final byte [] row,
915       final byte [] family, final byte [] qualifier, final byte [] value,
916       final Put put)
917   throws IOException {
918     return new ServerCallable<Boolean>(connection, tableName, row, operationTimeout) {
919           public Boolean call() throws IOException {
920             return server.checkAndPut(location.getRegionInfo().getRegionName(),
921                 row, family, qualifier, value, put) ? Boolean.TRUE : Boolean.FALSE;
922           }
923         }.withRetries();
924   }
925 
926 
927   /**
928    * {@inheritDoc}
929    */
930   @Override
931   public boolean checkAndDelete(final byte [] row,
932       final byte [] family, final byte [] qualifier, final byte [] value,
933       final Delete delete)
934   throws IOException {
935     return new ServerCallable<Boolean>(connection, tableName, row, operationTimeout) {
936           public Boolean call() throws IOException {
937             return server.checkAndDelete(
938                 location.getRegionInfo().getRegionName(),
939                 row, family, qualifier, value, delete)
940             ? Boolean.TRUE : Boolean.FALSE;
941           }
942         }.withRetries();
943   }
944 
945   /**
946    * {@inheritDoc}
947    */
948   @Override
949   public boolean exists(final Get get) throws IOException {
950     return new ServerCallable<Boolean>(connection, tableName, get.getRow(), operationTimeout) {
951           public Boolean call() throws IOException {
952             return server.
953                 exists(location.getRegionInfo().getRegionName(), get);
954           }
955         }.withRetries();
956   }
957 
958   /**
959    * {@inheritDoc}
960    */
961   @Override
962   public void flushCommits() throws IOException {
963     try {
964       Object[] results = new Object[writeBuffer.size()];
965       try {
966         this.connection.processBatch(writeBuffer, tableName, pool, results);
967       } catch (InterruptedException e) {
968         throw new IOException(e);
969       } finally {
970         // mutate list so that it is empty for complete success, or contains
971         // only failed records results are returned in the same order as the
972         // requests in list walk the list backwards, so we can remove from list
973         // without impacting the indexes of earlier members
974         for (int i = results.length - 1; i>=0; i--) {
975           if (results[i] instanceof Result) {
976             // successful Puts are removed from the list here.
977             writeBuffer.remove(i);
978           }
979         }
980       }
981     } finally {
982       if (clearBufferOnFail) {
983         writeBuffer.clear();
984         currentWriteBufferSize = 0;
985       } else {
986         // the write buffer was adjusted by processBatchOfPuts
987         currentWriteBufferSize = 0;
988         for (Put aPut : writeBuffer) {
989           currentWriteBufferSize += aPut.heapSize();
990         }
991       }
992     }
993   }
994 
995   /**
996    * {@inheritDoc}
997    */
998   @Override
999   public void close() throws IOException {
1000     if (this.closed) {
1001       return;
1002     }
1003     flushCommits();
1004     if (cleanupPoolOnClose) {
1005       this.pool.shutdown();
1006     }
1007     if (cleanupConnectionOnClose) {
1008       if (this.connection != null) {
1009         this.connection.close();
1010       }
1011     }
1012     this.closed = true;
1013   }
1014 
1015   // validate for well-formedness
1016   private void validatePut(final Put put) throws IllegalArgumentException{
1017     if (put.isEmpty()) {
1018       throw new IllegalArgumentException("No columns to insert");
1019     }
1020     if (maxKeyValueSize > 0) {
1021       for (List<KeyValue> list : put.getFamilyMap().values()) {
1022         for (KeyValue kv : list) {
1023           if (kv.getLength() > maxKeyValueSize) {
1024             throw new IllegalArgumentException("KeyValue size too large");
1025           }
1026         }
1027       }
1028     }
1029   }
1030 
1031   /**
1032    * {@inheritDoc}
1033    */
1034   @Override
1035   public RowLock lockRow(final byte [] row)
1036   throws IOException {
1037     return new ServerCallable<RowLock>(connection, tableName, row, operationTimeout) {
1038         public RowLock call() throws IOException {
1039           long lockId =
1040               server.lockRow(location.getRegionInfo().getRegionName(), row);
1041           return new RowLock(row,lockId);
1042         }
1043       }.withRetries();
1044   }
1045 
1046   /**
1047    * {@inheritDoc}
1048    */
1049   @Override
1050   public void unlockRow(final RowLock rl)
1051   throws IOException {
1052     new ServerCallable<Boolean>(connection, tableName, rl.getRow(), operationTimeout) {
1053         public Boolean call() throws IOException {
1054           server.unlockRow(location.getRegionInfo().getRegionName(),
1055               rl.getLockId());
1056           return null; // FindBugs NP_BOOLEAN_RETURN_NULL
1057         }
1058       }.withRetries();
1059   }
1060 
1061   /**
1062    * {@inheritDoc}
1063    */
1064   @Override
1065   public boolean isAutoFlush() {
1066     return autoFlush;
1067   }
1068 
1069   /**
1070    * See {@link #setAutoFlush(boolean, boolean)}
1071    *
1072    * @param autoFlush
1073    *          Whether or not to enable 'auto-flush'.
1074    */
1075   public void setAutoFlush(boolean autoFlush) {
1076     setAutoFlush(autoFlush, autoFlush);
1077   }
1078 
1079   /**
1080    * Turns 'auto-flush' on or off.
1081    * <p>
1082    * When enabled (default), {@link Put} operations don't get buffered/delayed
1083    * and are immediately executed. Failed operations are not retried. This is
1084    * slower but safer.
1085    * <p>
1086    * Turning off {@link #autoFlush} means that multiple {@link Put}s will be
1087    * accepted before any RPC is actually sent to do the write operations. If the
1088    * application dies before pending writes get flushed to HBase, data will be
1089    * lost.
1090    * <p>
1091    * When you turn {@link #autoFlush} off, you should also consider the
1092    * {@link #clearBufferOnFail} option. By default, asynchronous {@link Put}
1093    * requests will be retried on failure until successful. However, this can
1094    * pollute the writeBuffer and slow down batching performance. Additionally,
1095    * you may want to issue a number of Put requests and call
1096    * {@link #flushCommits()} as a barrier. In both use cases, consider setting
1097    * clearBufferOnFail to true to erase the buffer after {@link #flushCommits()}
1098    * has been called, regardless of success.
1099    *
1100    * @param autoFlush
1101    *          Whether or not to enable 'auto-flush'.
1102    * @param clearBufferOnFail
1103    *          Whether to keep Put failures in the writeBuffer
1104    * @see #flushCommits
1105    */
1106   public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
1107     this.autoFlush = autoFlush;
1108     this.clearBufferOnFail = autoFlush || clearBufferOnFail;
1109   }
1110 
1111   /**
1112    * Returns the maximum size in bytes of the write buffer for this HTable.
1113    * <p>
1114    * The default value comes from the configuration parameter
1115    * {@code hbase.client.write.buffer}.
1116    * @return The size of the write buffer in bytes.
1117    */
1118   public long getWriteBufferSize() {
1119     return writeBufferSize;
1120   }
1121 
1122   /**
1123    * Sets the size of the buffer in bytes.
1124    * <p>
1125    * If the new size is less than the current amount of data in the
1126    * write buffer, the buffer gets flushed.
1127    * @param writeBufferSize The new write buffer size, in bytes.
1128    * @throws IOException if a remote or network exception occurs.
1129    */
1130   public void setWriteBufferSize(long writeBufferSize) throws IOException {
1131     this.writeBufferSize = writeBufferSize;
1132     if(currentWriteBufferSize > writeBufferSize) {
1133       flushCommits();
1134     }
1135   }
1136 
1137   /**
1138    * Returns the write buffer.
1139    * @return The current write buffer.
1140    */
1141   public ArrayList<Put> getWriteBuffer() {
1142     return writeBuffer;
1143   }
1144 
1145   /**
1146    * The pool is used for mutli requests for this HTable
1147    * @return the pool used for mutli
1148    */
1149   ExecutorService getPool() {
1150     return this.pool;
1151   }
1152 
1153   /**
1154    * Enable or disable region cache prefetch for the table. It will be
1155    * applied for the given table's all HTable instances who share the same
1156    * connection. By default, the cache prefetch is enabled.
1157    * @param tableName name of table to configure.
1158    * @param enable Set to true to enable region cache prefetch. Or set to
1159    * false to disable it.
1160    * @throws IOException
1161    */
1162   public static void setRegionCachePrefetch(final byte[] tableName,
1163       final boolean enable) throws IOException {
1164     HConnectionManager.execute(new HConnectable<Void>(HBaseConfiguration
1165         .create()) {
1166       @Override
1167       public Void connect(HConnection connection) throws IOException {
1168         connection.setRegionCachePrefetch(tableName, enable);
1169         return null;
1170       }
1171     });
1172   }
1173 
1174   /**
1175    * Enable or disable region cache prefetch for the table. It will be
1176    * applied for the given table's all HTable instances who share the same
1177    * connection. By default, the cache prefetch is enabled.
1178    * @param conf The Configuration object to use.
1179    * @param tableName name of table to configure.
1180    * @param enable Set to true to enable region cache prefetch. Or set to
1181    * false to disable it.
1182    * @throws IOException
1183    */
1184   public static void setRegionCachePrefetch(final Configuration conf,
1185       final byte[] tableName, final boolean enable) throws IOException {
1186     HConnectionManager.execute(new HConnectable<Void>(conf) {
1187       @Override
1188       public Void connect(HConnection connection) throws IOException {
1189         connection.setRegionCachePrefetch(tableName, enable);
1190         return null;
1191       }
1192     });
1193   }
1194 
1195   /**
1196    * Check whether region cache prefetch is enabled or not for the table.
1197    * @param conf The Configuration object to use.
1198    * @param tableName name of table to check
1199    * @return true if table's region cache prefecth is enabled. Otherwise
1200    * it is disabled.
1201    * @throws IOException
1202    */
1203   public static boolean getRegionCachePrefetch(final Configuration conf,
1204       final byte[] tableName) throws IOException {
1205     return HConnectionManager.execute(new HConnectable<Boolean>(conf) {
1206       @Override
1207       public Boolean connect(HConnection connection) throws IOException {
1208         return connection.getRegionCachePrefetch(tableName);
1209       }
1210     });
1211   }
1212 
1213   /**
1214    * Check whether region cache prefetch is enabled or not for the table.
1215    * @param tableName name of table to check
1216    * @return true if table's region cache prefecth is enabled. Otherwise
1217    * it is disabled.
1218    * @throws IOException
1219    */
1220   public static boolean getRegionCachePrefetch(final byte[] tableName) throws IOException {
1221     return HConnectionManager.execute(new HConnectable<Boolean>(
1222         HBaseConfiguration.create()) {
1223       @Override
1224       public Boolean connect(HConnection connection) throws IOException {
1225         return connection.getRegionCachePrefetch(tableName);
1226       }
1227     });
1228  }
1229 
1230   /**
1231    * Explicitly clears the region cache to fetch the latest value from META.
1232    * This is a power user function: avoid unless you know the ramifications.
1233    */
1234   public void clearRegionCache() {
1235     this.connection.clearRegionCache();
1236   }
1237 
1238   /**
1239    * {@inheritDoc}
1240    */
1241   @Override
1242   public <T extends CoprocessorProtocol> T coprocessorProxy(
1243       Class<T> protocol, byte[] row) {
1244     return (T)Proxy.newProxyInstance(this.getClass().getClassLoader(),
1245         new Class[]{protocol},
1246         new ExecRPCInvoker(configuration,
1247             connection,
1248             protocol,
1249             tableName,
1250             row));
1251   }
1252 
1253   /**
1254    * {@inheritDoc}
1255    */
1256   @Override
1257   public <T extends CoprocessorProtocol, R> Map<byte[],R> coprocessorExec(
1258       Class<T> protocol, byte[] startKey, byte[] endKey,
1259       Batch.Call<T,R> callable)
1260       throws IOException, Throwable {
1261 
1262     final Map<byte[],R> results =  Collections.synchronizedMap(new TreeMap<byte[],R>(
1263         Bytes.BYTES_COMPARATOR));
1264     coprocessorExec(protocol, startKey, endKey, callable,
1265         new Batch.Callback<R>(){
1266       public void update(byte[] region, byte[] row, R value) {
1267         results.put(region, value);
1268       }
1269     });
1270     return results;
1271   }
1272 
1273   /**
1274    * {@inheritDoc}
1275    */
1276   @Override
1277   public <T extends CoprocessorProtocol, R> void coprocessorExec(
1278       Class<T> protocol, byte[] startKey, byte[] endKey,
1279       Batch.Call<T,R> callable, Batch.Callback<R> callback)
1280       throws IOException, Throwable {
1281 
1282     // get regions covered by the row range
1283     List<byte[]> keys = getStartKeysInRange(startKey, endKey);
1284     connection.processExecs(protocol, keys, tableName, pool, callable,
1285         callback);
1286   }
1287 
1288   private List<byte[]> getStartKeysInRange(byte[] start, byte[] end)
1289   throws IOException {
1290     if (start == null) {
1291       start = HConstants.EMPTY_START_ROW;
1292     }
1293     if (end == null) {
1294       end = HConstants.EMPTY_END_ROW;
1295     }
1296     return getKeysAndRegionsInRange(start, end, true).getFirst();
1297   }
1298 
1299   public void setOperationTimeout(int operationTimeout) {
1300     this.operationTimeout = operationTimeout;
1301   }
1302 
1303   public int getOperationTimeout() {
1304     return operationTimeout;
1305   }
1306 
1307 }