View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  import java.net.SocketTimeoutException;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.atomic.AtomicInteger;
32  import java.util.concurrent.atomic.AtomicReference;
33  import java.util.regex.Pattern;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.Abortable;
39  import org.apache.hadoop.hbase.ClusterStatus;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HBaseIOException;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HRegionLocation;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.MasterNotRunningException;
48  import org.apache.hadoop.hbase.NamespaceDescriptor;
49  import org.apache.hadoop.hbase.NotServingRegionException;
50  import org.apache.hadoop.hbase.RegionException;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.TableExistsException;
53  import org.apache.hadoop.hbase.TableName;
54  import org.apache.hadoop.hbase.TableNotDisabledException;
55  import org.apache.hadoop.hbase.TableNotEnabledException;
56  import org.apache.hadoop.hbase.TableNotFoundException;
57  import org.apache.hadoop.hbase.UnknownRegionException;
58  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
59  import org.apache.hadoop.hbase.catalog.CatalogTracker;
60  import org.apache.hadoop.hbase.catalog.MetaReader;
61  import org.apache.hadoop.hbase.classification.InterfaceAudience;
62  import org.apache.hadoop.hbase.classification.InterfaceStability;
63  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
64  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
65  import org.apache.hadoop.hbase.exceptions.DeserializationException;
66  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
67  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
68  import org.apache.hadoop.hbase.ipc.MasterCoprocessorRpcChannel;
69  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
70  import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel;
71  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
72  import org.apache.hadoop.hbase.protobuf.RequestConverter;
73  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
86  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
87  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
88  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
89  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
90  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
91  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
92  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
93  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
94  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnRequest;
98  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotRequest;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRequest;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureResponse;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
117 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
118 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
119 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
120 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest;
121 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest;
122 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
124 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRequest;
125 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
126 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
127 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
128 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
129 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
130 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest;
131 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
132 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
133 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableRequest;
134 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
135 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
136 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
137 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
138 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
139 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
140 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
141 import org.apache.hadoop.hbase.util.Addressing;
142 import org.apache.hadoop.hbase.util.Bytes;
143 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
144 import org.apache.hadoop.hbase.util.Pair;
145 import org.apache.hadoop.ipc.RemoteException;
146 import org.apache.hadoop.util.StringUtils;
147 import org.apache.zookeeper.KeeperException;
148 
149 import com.google.common.annotations.VisibleForTesting;
150 import com.google.protobuf.ByteString;
151 import com.google.protobuf.ServiceException;
152 
153 /**
154  * Provides an interface to manage HBase database table metadata + general
155  * administrative functions.  Use HBaseAdmin to create, drop, list, enable and
156  * disable tables. Use it also to add and drop table column families.
157  *
158  * <p>See {@link HTable} to add, update, and delete data from an individual table.
159  * <p>Currently HBaseAdmin instances are not expected to be long-lived.  For
160  * example, an HBaseAdmin instance will not ride over a Master restart.
161  */
162 @InterfaceAudience.Public
163 @InterfaceStability.Evolving
164 public class HBaseAdmin implements Abortable, Closeable {
165   private static final Log LOG = LogFactory.getLog(HBaseAdmin.class);
166 
167   // We use the implementation class rather then the interface because we
168   //  need the package protected functions to get the connection to master
169   private HConnection connection;
170 
171   private volatile Configuration conf;
172   private final long pause;
173   private final int numRetries;
174   // Some operations can take a long time such as disable of big table.
175   // numRetries is for 'normal' stuff... Multiply by this factor when
176   // want to wait a long time.
177   private final int retryLongerMultiplier;
178   private boolean aborted;
179   private boolean cleanupConnectionOnClose = false; // close the connection in close()
180   private boolean closed = false;
181 
182   private RpcRetryingCallerFactory rpcCallerFactory;
183 
184   /**
185    * Constructor.
186    * See {@link #HBaseAdmin(HConnection connection)}
187    *
188    * @param c Configuration object. Copied internally.
189    */
190   public HBaseAdmin(Configuration c)
191   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
192     // Will not leak connections, as the new implementation of the constructor
193     // does not throw exceptions anymore.
194     this(HConnectionManager.getConnection(new Configuration(c)));
195     this.cleanupConnectionOnClose = true;
196   }
197 
198  /**
199   * Constructor for externally managed HConnections.
200   * The connection to master will be created when required by admin functions.
201   *
202   * @param connection The HConnection instance to use
203   * @throws MasterNotRunningException, ZooKeeperConnectionException are not
204   *  thrown anymore but kept into the interface for backward api compatibility
205   */
206   public HBaseAdmin(HConnection connection)
207       throws MasterNotRunningException, ZooKeeperConnectionException {
208     this.conf = connection.getConfiguration();
209     this.connection = connection;
210 
211     this.pause = this.conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
212         HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
213     this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
214         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
215     this.retryLongerMultiplier = this.conf.getInt(
216         "hbase.client.retries.longer.multiplier", 10);
217     this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf,
218       connection.getStatisticsTracker());
219   }
220 
221   /**
222    * @return A new CatalogTracker instance; call {@link #cleanupCatalogTracker(CatalogTracker)}
223    * to cleanup the returned catalog tracker.
224    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
225    * @throws IOException
226    * @see #cleanupCatalogTracker(CatalogTracker)
227    */
228   @VisibleForTesting
229   synchronized CatalogTracker getCatalogTracker()
230   throws ZooKeeperConnectionException, IOException {
231     boolean succeeded = false;
232     CatalogTracker ct = null;
233     try {
234       ct = new CatalogTracker(this.conf);
235       startCatalogTracker(ct);
236       succeeded = true;
237     } catch (InterruptedException e) {
238       // Let it out as an IOE for now until we redo all so tolerate IEs
239       throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
240     } finally {
241       // If we did not succeed but created a catalogtracker, clean it up. CT has a ZK instance
242       // in it and we'll leak if we don't do the 'stop'.
243       if (!succeeded && ct != null) {
244         try {
245           ct.stop();
246         } catch (RuntimeException re) {
247           LOG.error("Failed to clean up HBase's internal catalog tracker after a failed initialization. " +
248             "We may have leaked network connections to ZooKeeper; they won't be cleaned up until " +
249             "the JVM exits. If you see a large number of stale connections to ZooKeeper this is likely " +
250             "the cause. The following exception details will be needed for assistance from the " +
251             "HBase community.", re);
252         }
253         ct = null;
254       }
255     }
256     return ct;
257   }
258 
259   @VisibleForTesting
260   CatalogTracker startCatalogTracker(final CatalogTracker ct)
261   throws IOException, InterruptedException {
262     ct.start();
263     return ct;
264   }
265 
266   @VisibleForTesting
267   void cleanupCatalogTracker(final CatalogTracker ct) {
268     ct.stop();
269   }
270 
271   @Override
272   public void abort(String why, Throwable e) {
273     // Currently does nothing but throw the passed message and exception
274     this.aborted = true;
275     throw new RuntimeException(why, e);
276   }
277 
278   @Override
279   public boolean isAborted(){
280     return this.aborted;
281   }
282 
283   /** @return HConnection used by this object. */
284   public HConnection getConnection() {
285     return connection;
286   }
287 
288   /** @return - true if the master server is running. Throws an exception
289    *  otherwise.
290    * @throws ZooKeeperConnectionException
291    * @throws MasterNotRunningException
292    */
293   public boolean isMasterRunning()
294   throws MasterNotRunningException, ZooKeeperConnectionException {
295     return connection.isMasterRunning();
296   }
297 
298   /**
299    * @param tableName Table to check.
300    * @return True if table exists already.
301    * @throws IOException
302    */
303   public boolean tableExists(final TableName tableName)
304   throws IOException {
305     boolean b = false;
306     CatalogTracker ct = getCatalogTracker();
307     try {
308       b = MetaReader.tableExists(ct, tableName);
309     } finally {
310       cleanupCatalogTracker(ct);
311     }
312     return b;
313   }
314 
315   public boolean tableExists(final byte[] tableName)
316   throws IOException {
317     return tableExists(TableName.valueOf(tableName));
318   }
319 
320   public boolean tableExists(final String tableName)
321   throws IOException {
322     return tableExists(TableName.valueOf(tableName));
323   }
324 
325   /**
326    * List all the userspace tables.  In other words, scan the hbase:meta table.
327    *
328    * If we wanted this to be really fast, we could implement a special
329    * catalog table that just contains table names and their descriptors.
330    * Right now, it only exists as part of the hbase:meta table's region info.
331    *
332    * @return - returns an array of HTableDescriptors
333    * @throws IOException if a remote or network exception occurs
334    */
335   public HTableDescriptor[] listTables() throws IOException {
336     return this.connection.listTables();
337   }
338 
339   /**
340    * List all the userspace tables matching the given pattern.
341    *
342    * @param pattern The compiled regular expression to match against
343    * @return - returns an array of HTableDescriptors
344    * @throws IOException if a remote or network exception occurs
345    * @see #listTables()
346    */
347   public HTableDescriptor[] listTables(Pattern pattern) throws IOException {
348     List<HTableDescriptor> matched = new LinkedList<HTableDescriptor>();
349     HTableDescriptor[] tables = listTables();
350     for (HTableDescriptor table : tables) {
351       if (pattern.matcher(table.getTableName().getNameAsString()).matches()) {
352         matched.add(table);
353       }
354     }
355     return matched.toArray(new HTableDescriptor[matched.size()]);
356   }
357 
358   /**
359    * List all the userspace tables matching the given regular expression.
360    *
361    * @param regex The regular expression to match against
362    * @return - returns an array of HTableDescriptors
363    * @throws IOException if a remote or network exception occurs
364    * @see #listTables(java.util.regex.Pattern)
365    */
366   public HTableDescriptor[] listTables(String regex) throws IOException {
367     return listTables(Pattern.compile(regex));
368   }
369 
370   /**
371    * List all of the names of userspace tables.
372    * @return String[] table names
373    * @throws IOException if a remote or network exception occurs
374    */
375   @Deprecated
376   public String[] getTableNames() throws IOException {
377     return this.connection.getTableNames();
378   }
379 
380   /**
381    * List all of the names of userspace tables matching the given regular expression.
382    * @param pattern The regular expression to match against
383    * @return String[] table names
384    * @throws IOException if a remote or network exception occurs
385    */
386   @Deprecated
387   public String[] getTableNames(Pattern pattern) throws IOException {
388     List<String> matched = new ArrayList<String>();
389     for (String name: this.connection.getTableNames()) {
390       if (pattern.matcher(name).matches()) {
391         matched.add(name);
392       }
393     }
394     return matched.toArray(new String[matched.size()]);
395   }
396 
397   /**
398    * List all of the names of userspace tables matching the given regular expression.
399    * @param regex The regular expression to match against
400    * @return String[] table names
401    * @throws IOException if a remote or network exception occurs
402    */
403   @Deprecated
404   public String[] getTableNames(String regex) throws IOException {
405     return getTableNames(Pattern.compile(regex));
406   }
407 
408   /**
409    * List all of the names of userspace tables.
410    * @return TableName[] table names
411    * @throws IOException if a remote or network exception occurs
412    */
413   public TableName[] listTableNames() throws IOException {
414     return this.connection.listTableNames();
415   }
416 
417   /**
418    * Method for getting the tableDescriptor
419    * @param tableName as a byte []
420    * @return the tableDescriptor
421    * @throws TableNotFoundException
422    * @throws IOException if a remote or network exception occurs
423    */
424   public HTableDescriptor getTableDescriptor(final TableName tableName)
425   throws TableNotFoundException, IOException {
426     return this.connection.getHTableDescriptor(tableName);
427   }
428 
429   public HTableDescriptor getTableDescriptor(final byte[] tableName)
430   throws TableNotFoundException, IOException {
431     return getTableDescriptor(TableName.valueOf(tableName));
432   }
433 
434   private long getPauseTime(int tries) {
435     int triesCount = tries;
436     if (triesCount >= HConstants.RETRY_BACKOFF.length) {
437       triesCount = HConstants.RETRY_BACKOFF.length - 1;
438     }
439     return this.pause * HConstants.RETRY_BACKOFF[triesCount];
440   }
441 
442   /**
443    * Creates a new table.
444    * Synchronous operation.
445    *
446    * @param desc table descriptor for table
447    *
448    * @throws IllegalArgumentException if the table name is reserved
449    * @throws MasterNotRunningException if master is not running
450    * @throws TableExistsException if table already exists (If concurrent
451    * threads, the table may have been created between test-for-existence
452    * and attempt-at-creation).
453    * @throws IOException if a remote or network exception occurs
454    */
455   public void createTable(HTableDescriptor desc)
456   throws IOException {
457     createTable(desc, null);
458   }
459 
460   /**
461    * Creates a new table with the specified number of regions.  The start key
462    * specified will become the end key of the first region of the table, and
463    * the end key specified will become the start key of the last region of the
464    * table (the first region has a null start key and the last region has a
465    * null end key).
466    *
467    * BigInteger math will be used to divide the key range specified into
468    * enough segments to make the required number of total regions.
469    *
470    * Synchronous operation.
471    *
472    * @param desc table descriptor for table
473    * @param startKey beginning of key range
474    * @param endKey end of key range
475    * @param numRegions the total number of regions to create
476    *
477    * @throws IllegalArgumentException if the table name is reserved
478    * @throws MasterNotRunningException if master is not running
479    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
480    * threads, the table may have been created between test-for-existence
481    * and attempt-at-creation).
482    * @throws IOException
483    */
484   public void createTable(HTableDescriptor desc, byte [] startKey,
485       byte [] endKey, int numRegions)
486   throws IOException {
487     if(numRegions < 3) {
488       throw new IllegalArgumentException("Must create at least three regions");
489     } else if(Bytes.compareTo(startKey, endKey) >= 0) {
490       throw new IllegalArgumentException("Start key must be smaller than end key");
491     }
492     if (numRegions == 3) {
493       createTable(desc, new byte[][]{startKey, endKey});
494       return;
495     }
496     byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
497     if(splitKeys == null || splitKeys.length != numRegions - 1) {
498       throw new IllegalArgumentException("Unable to split key range into enough regions");
499     }
500     createTable(desc, splitKeys);
501   }
502 
503   /**
504    * Creates a new table with an initial set of empty regions defined by the
505    * specified split keys.  The total number of regions created will be the
506    * number of split keys plus one. Synchronous operation.
507    * Note : Avoid passing empty split key.
508    *
509    * @param desc table descriptor for table
510    * @param splitKeys array of split keys for the initial regions of the table
511    *
512    * @throws IllegalArgumentException if the table name is reserved, if the split keys
513    * are repeated and if the split key has empty byte array.
514    * @throws MasterNotRunningException if master is not running
515    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
516    * threads, the table may have been created between test-for-existence
517    * and attempt-at-creation).
518    * @throws IOException
519    */
520   public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
521   throws IOException {
522     try {
523       createTableAsync(desc, splitKeys);
524     } catch (SocketTimeoutException ste) {
525       LOG.warn("Creating " + desc.getTableName() + " took too long", ste);
526     }
527     int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
528     int prevRegCount = 0;
529     boolean doneWithMetaScan = false;
530     for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier;
531       ++tries) {
532       if (!doneWithMetaScan) {
533         // Wait for new table to come on-line
534         final AtomicInteger actualRegCount = new AtomicInteger(0);
535         MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
536           @Override
537           public boolean processRow(Result rowResult) throws IOException {
538             HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
539             if (info == null) {
540               LOG.warn("No serialized HRegionInfo in " + rowResult);
541               return true;
542             }
543             if (!info.getTable().equals(desc.getTableName())) {
544               return false;
545             }
546             ServerName serverName = HRegionInfo.getServerName(rowResult);
547             // Make sure that regions are assigned to server
548             if (!(info.isOffline() || info.isSplit()) && serverName != null
549                 && serverName.getHostAndPort() != null) {
550               actualRegCount.incrementAndGet();
551             }
552             return true;
553           }
554         };
555         MetaScanner.metaScan(conf, connection, visitor, desc.getTableName());
556         if (actualRegCount.get() < numRegs) {
557           if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
558             throw new RegionOfflineException("Only " + actualRegCount.get() +
559               " of " + numRegs + " regions are online; retries exhausted.");
560           }
561           try { // Sleep
562             Thread.sleep(getPauseTime(tries));
563           } catch (InterruptedException e) {
564             throw new InterruptedIOException("Interrupted when opening" +
565               " regions; " + actualRegCount.get() + " of " + numRegs +
566               " regions processed so far");
567           }
568           if (actualRegCount.get() > prevRegCount) { // Making progress
569             prevRegCount = actualRegCount.get();
570             tries = -1;
571           }
572         } else {
573           doneWithMetaScan = true;
574           tries = -1;
575         }
576       } else if (isTableEnabled(desc.getTableName())) {
577         return;
578       } else {
579         try { // Sleep
580           Thread.sleep(getPauseTime(tries));
581         } catch (InterruptedException e) {
582           throw new InterruptedIOException("Interrupted when waiting" +
583             " for table to be enabled; meta scan was done");
584         }
585       }
586     }
587     throw new TableNotEnabledException(
588       "Retries exhausted while still waiting for table: "
589       + desc.getTableName() + " to be enabled");
590   }
591 
592   /**
593    * Creates a new table but does not block and wait for it to come online.
594    * Asynchronous operation.  To check if the table exists, use
595    * {@link #isTableAvailable} -- it is not safe to create an HTable
596    * instance to this table before it is available.
597    * Note : Avoid passing empty split key.
598    * @param desc table descriptor for table
599    *
600    * @throws IllegalArgumentException Bad table name, if the split keys
601    * are repeated and if the split key has empty byte array.
602    * @throws MasterNotRunningException if master is not running
603    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
604    * threads, the table may have been created between test-for-existence
605    * and attempt-at-creation).
606    * @throws IOException
607    */
608   public void createTableAsync(
609     final HTableDescriptor desc, final byte [][] splitKeys)
610   throws IOException {
611     if(desc.getTableName() == null) {
612       throw new IllegalArgumentException("TableName cannot be null");
613     }
614     if(splitKeys != null && splitKeys.length > 0) {
615       Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
616       // Verify there are no duplicate split keys
617       byte [] lastKey = null;
618       for(byte [] splitKey : splitKeys) {
619         if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) {
620           throw new IllegalArgumentException(
621               "Empty split key must not be passed in the split keys.");
622         }
623         if(lastKey != null && Bytes.equals(splitKey, lastKey)) {
624           throw new IllegalArgumentException("All split keys must be unique, " +
625             "found duplicate: " + Bytes.toStringBinary(splitKey) +
626             ", " + Bytes.toStringBinary(lastKey));
627         }
628         lastKey = splitKey;
629       }
630     }
631 
632     executeCallable(new MasterCallable<Void>(getConnection()) {
633       @Override
634       public Void call() throws ServiceException {
635         CreateTableRequest request = RequestConverter.buildCreateTableRequest(desc, splitKeys);
636         master.createTable(null, request);
637         return null;
638       }
639     });
640   }
641 
642   public void deleteTable(final String tableName) throws IOException {
643     deleteTable(TableName.valueOf(tableName));
644   }
645 
646   public void deleteTable(final byte[] tableName) throws IOException {
647     deleteTable(TableName.valueOf(tableName));
648   }
649 
650   /**
651    * Deletes a table.
652    * Synchronous operation.
653    *
654    * @param tableName name of table to delete
655    * @throws IOException if a remote or network exception occurs
656    */
657   public void deleteTable(final TableName tableName) throws IOException {
658     boolean tableExists = true;
659 
660     executeCallable(new MasterCallable<Void>(getConnection()) {
661       @Override
662       public Void call() throws ServiceException {
663         DeleteTableRequest req = RequestConverter.buildDeleteTableRequest(tableName);
664         master.deleteTable(null,req);
665         return null;
666       }
667     });
668 
669     int failures = 0;
670     // Wait until all regions deleted
671     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
672       try {
673         HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName);
674         Scan scan = MetaReader.getScanForTableName(tableName);
675         scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
676         ScanRequest request = RequestConverter.buildScanRequest(
677           firstMetaServer.getRegionInfo().getRegionName(), scan, 1, true);
678         Result[] values = null;
679         // Get a batch at a time.
680         ClientService.BlockingInterface server = connection.getClient(firstMetaServer
681             .getServerName());
682         PayloadCarryingRpcController controller = new PayloadCarryingRpcController();
683         try {
684           controller.setPriority(tableName);
685           ScanResponse response = server.scan(controller, request);
686           values = ResponseConverter.getResults(controller.cellScanner(), response);
687         } catch (ServiceException se) {
688           throw ProtobufUtil.getRemoteException(se);
689         }
690 
691         // let us wait until hbase:meta table is updated and
692         // HMaster removes the table from its HTableDescriptors
693         if (values == null || values.length == 0) {
694           tableExists = false;
695           GetTableDescriptorsResponse htds;
696           MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
697           try {
698             GetTableDescriptorsRequest req =
699               RequestConverter.buildGetTableDescriptorsRequest(tableName);
700             htds = master.getTableDescriptors(null, req);
701           } catch (ServiceException se) {
702             throw ProtobufUtil.getRemoteException(se);
703           } finally {
704             master.close();
705           }
706           tableExists = !htds.getTableSchemaList().isEmpty();
707           if (!tableExists) {
708             break;
709           }
710         }
711       } catch (IOException ex) {
712         failures++;
713         if(failures == numRetries - 1) {           // no more tries left
714           if (ex instanceof RemoteException) {
715             throw ((RemoteException) ex).unwrapRemoteException();
716           } else {
717             throw ex;
718           }
719         }
720       }
721       try {
722         Thread.sleep(getPauseTime(tries));
723       } catch (InterruptedException e) {
724         // continue
725       }
726     }
727 
728     if (tableExists) {
729       throw new IOException("Retries exhausted, it took too long to wait"+
730         " for the table " + tableName + " to be deleted.");
731     }
732     // Delete cached information to prevent clients from using old locations
733     this.connection.clearRegionCache(tableName);
734     LOG.info("Deleted " + tableName);
735   }
736 
737   /**
738    * Deletes tables matching the passed in pattern and wait on completion.
739    *
740    * Warning: Use this method carefully, there is no prompting and the effect is
741    * immediate. Consider using {@link #listTables(java.lang.String)} and
742    * {@link #deleteTable(byte[])}
743    *
744    * @param regex The regular expression to match table names against
745    * @return Table descriptors for tables that couldn't be deleted
746    * @throws IOException
747    * @see #deleteTables(java.util.regex.Pattern)
748    * @see #deleteTable(java.lang.String)
749    */
750   public HTableDescriptor[] deleteTables(String regex) throws IOException {
751     return deleteTables(Pattern.compile(regex));
752   }
753 
754   /**
755    * Delete tables matching the passed in pattern and wait on completion.
756    *
757    * Warning: Use this method carefully, there is no prompting and the effect is
758    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
759    * {@link #deleteTable(byte[])}
760    *
761    * @param pattern The pattern to match table names against
762    * @return Table descriptors for tables that couldn't be deleted
763    * @throws IOException
764    */
765   public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException {
766     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
767     for (HTableDescriptor table : listTables(pattern)) {
768       try {
769         deleteTable(table.getTableName());
770       } catch (IOException ex) {
771         LOG.info("Failed to delete table " + table.getTableName(), ex);
772         failed.add(table);
773       }
774     }
775     return failed.toArray(new HTableDescriptor[failed.size()]);
776   }
777 
778 
779   /**
780    * Enable a table.  May timeout.  Use {@link #enableTableAsync(byte[])}
781    * and {@link #isTableEnabled(byte[])} instead.
782    * The table has to be in disabled state for it to be enabled.
783    * @param tableName name of the table
784    * @throws IOException if a remote or network exception occurs
785    * There could be couple types of IOException
786    * TableNotFoundException means the table doesn't exist.
787    * TableNotDisabledException means the table isn't in disabled state.
788    * @see #isTableEnabled(byte[])
789    * @see #disableTable(byte[])
790    * @see #enableTableAsync(byte[])
791    */
792   public void enableTable(final TableName tableName)
793   throws IOException {
794     enableTableAsync(tableName);
795 
796     // Wait until all regions are enabled
797     waitUntilTableIsEnabled(tableName);
798 
799     LOG.info("Enabled table " + tableName);
800   }
801 
802   public void enableTable(final byte[] tableName)
803   throws IOException {
804     enableTable(TableName.valueOf(tableName));
805   }
806 
807   public void enableTable(final String tableName)
808   throws IOException {
809     enableTable(TableName.valueOf(tableName));
810   }
811 
812   /**
813    * Wait for the table to be enabled and available
814    * If enabling the table exceeds the retry period, an exception is thrown.
815    * @param tableName name of the table
816    * @throws IOException if a remote or network exception occurs or
817    *    table is not enabled after the retries period.
818    */
819   private void waitUntilTableIsEnabled(final TableName tableName) throws IOException {
820     boolean enabled = false;
821     long start = EnvironmentEdgeManager.currentTimeMillis();
822     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
823       try {
824         enabled = isTableEnabled(tableName);
825       } catch (TableNotFoundException tnfe) {
826         // wait for table to be created
827         enabled = false;
828       }
829       enabled = enabled && isTableAvailable(tableName);
830       if (enabled) {
831         break;
832       }
833       long sleep = getPauseTime(tries);
834       if (LOG.isDebugEnabled()) {
835         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
836           "enabled in " + tableName);
837       }
838       try {
839         Thread.sleep(sleep);
840       } catch (InterruptedException e) {
841         // Do this conversion rather than let it out because do not want to
842         // change the method signature.
843         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
844       }
845     }
846     if (!enabled) {
847       long msec = EnvironmentEdgeManager.currentTimeMillis() - start;
848       throw new IOException("Table '" + tableName +
849         "' not yet enabled, after " + msec + "ms.");
850     }
851   }
852 
853   /**
854    * Brings a table on-line (enables it).  Method returns immediately though
855    * enable of table may take some time to complete, especially if the table
856    * is large (All regions are opened as part of enabling process).  Check
857    * {@link #isTableEnabled(byte[])} to learn when table is fully online.  If
858    * table is taking too long to online, check server logs.
859    * @param tableName
860    * @throws IOException
861    * @since 0.90.0
862    */
863   public void enableTableAsync(final TableName tableName)
864   throws IOException {
865     TableName.isLegalFullyQualifiedTableName(tableName.getName());
866     executeCallable(new MasterCallable<Void>(getConnection()) {
867       @Override
868       public Void call() throws ServiceException {
869         LOG.info("Started enable of " + tableName);
870         EnableTableRequest req = RequestConverter.buildEnableTableRequest(tableName);
871         master.enableTable(null,req);
872         return null;
873       }
874     });
875   }
876 
877   public void enableTableAsync(final byte[] tableName)
878   throws IOException {
879     enableTable(TableName.valueOf(tableName));
880   }
881 
882   public void enableTableAsync(final String tableName)
883   throws IOException {
884     enableTableAsync(TableName.valueOf(tableName));
885   }
886 
887   /**
888    * Enable tables matching the passed in pattern and wait on completion.
889    *
890    * Warning: Use this method carefully, there is no prompting and the effect is
891    * immediate. Consider using {@link #listTables(java.lang.String)} and
892    * {@link #enableTable(byte[])}
893    *
894    * @param regex The regular expression to match table names against
895    * @throws IOException
896    * @see #enableTables(java.util.regex.Pattern)
897    * @see #enableTable(java.lang.String)
898    */
899   public HTableDescriptor[] enableTables(String regex) throws IOException {
900     return enableTables(Pattern.compile(regex));
901   }
902 
903   /**
904    * Enable tables matching the passed in pattern and wait on completion.
905    *
906    * Warning: Use this method carefully, there is no prompting and the effect is
907    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
908    * {@link #enableTable(byte[])}
909    *
910    * @param pattern The pattern to match table names against
911    * @throws IOException
912    */
913   public HTableDescriptor[] enableTables(Pattern pattern) throws IOException {
914     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
915     for (HTableDescriptor table : listTables(pattern)) {
916       if (isTableDisabled(table.getTableName())) {
917         try {
918           enableTable(table.getTableName());
919         } catch (IOException ex) {
920           LOG.info("Failed to enable table " + table.getTableName(), ex);
921           failed.add(table);
922         }
923       }
924     }
925     return failed.toArray(new HTableDescriptor[failed.size()]);
926   }
927 
928   /**
929    * Starts the disable of a table.  If it is being served, the master
930    * will tell the servers to stop serving it.  This method returns immediately.
931    * The disable of a table can take some time if the table is large (all
932    * regions are closed as part of table disable operation).
933    * Call {@link #isTableDisabled(byte[])} to check for when disable completes.
934    * If table is taking too long to online, check server logs.
935    * @param tableName name of table
936    * @throws IOException if a remote or network exception occurs
937    * @see #isTableDisabled(byte[])
938    * @see #isTableEnabled(byte[])
939    * @since 0.90.0
940    */
941   public void disableTableAsync(final TableName tableName) throws IOException {
942     TableName.isLegalFullyQualifiedTableName(tableName.getName());
943     executeCallable(new MasterCallable<Void>(getConnection()) {
944       @Override
945       public Void call() throws ServiceException {
946         LOG.info("Started disable of " + tableName);
947         DisableTableRequest req = RequestConverter.buildDisableTableRequest(tableName);
948         master.disableTable(null,req);
949         return null;
950       }
951     });
952   }
953 
954   public void disableTableAsync(final byte[] tableName) throws IOException {
955     disableTableAsync(TableName.valueOf(tableName));
956   }
957 
958   public void disableTableAsync(final String tableName) throws IOException {
959     disableTableAsync(TableName.valueOf(tableName));
960   }
961 
962   /**
963    * Disable table and wait on completion.  May timeout eventually.  Use
964    * {@link #disableTableAsync(byte[])} and {@link #isTableDisabled(String)}
965    * instead.
966    * The table has to be in enabled state for it to be disabled.
967    * @param tableName
968    * @throws IOException
969    * There could be couple types of IOException
970    * TableNotFoundException means the table doesn't exist.
971    * TableNotEnabledException means the table isn't in enabled state.
972    */
973   public void disableTable(final TableName tableName)
974   throws IOException {
975     disableTableAsync(tableName);
976     // Wait until table is disabled
977     boolean disabled = false;
978     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
979       disabled = isTableDisabled(tableName);
980       if (disabled) {
981         break;
982       }
983       long sleep = getPauseTime(tries);
984       if (LOG.isDebugEnabled()) {
985         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
986           "disabled in " + tableName);
987       }
988       try {
989         Thread.sleep(sleep);
990       } catch (InterruptedException e) {
991         // Do this conversion rather than let it out because do not want to
992         // change the method signature.
993         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
994       }
995     }
996     if (!disabled) {
997       throw new RegionException("Retries exhausted, it took too long to wait"+
998         " for the table " + tableName + " to be disabled.");
999     }
1000     LOG.info("Disabled " + tableName);
1001   }
1002 
1003   public void disableTable(final byte[] tableName)
1004   throws IOException {
1005     disableTable(TableName.valueOf(tableName));
1006   }
1007 
1008   public void disableTable(final String tableName)
1009   throws IOException {
1010     disableTable(TableName.valueOf(tableName));
1011   }
1012 
1013   /**
1014    * Disable tables matching the passed in pattern and wait on completion.
1015    *
1016    * Warning: Use this method carefully, there is no prompting and the effect is
1017    * immediate. Consider using {@link #listTables(java.lang.String)} and
1018    * {@link #disableTable(byte[])}
1019    *
1020    * @param regex The regular expression to match table names against
1021    * @return Table descriptors for tables that couldn't be disabled
1022    * @throws IOException
1023    * @see #disableTables(java.util.regex.Pattern)
1024    * @see #disableTable(java.lang.String)
1025    */
1026   public HTableDescriptor[] disableTables(String regex) throws IOException {
1027     return disableTables(Pattern.compile(regex));
1028   }
1029 
1030   /**
1031    * Disable tables matching the passed in pattern and wait on completion.
1032    *
1033    * Warning: Use this method carefully, there is no prompting and the effect is
1034    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
1035    * {@link #disableTable(byte[])}
1036    *
1037    * @param pattern The pattern to match table names against
1038    * @return Table descriptors for tables that couldn't be disabled
1039    * @throws IOException
1040    */
1041   public HTableDescriptor[] disableTables(Pattern pattern) throws IOException {
1042     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
1043     for (HTableDescriptor table : listTables(pattern)) {
1044       if (isTableEnabled(table.getTableName())) {
1045         try {
1046           disableTable(table.getTableName());
1047         } catch (IOException ex) {
1048           LOG.info("Failed to disable table " + table.getTableName(), ex);
1049           failed.add(table);
1050         }
1051       }
1052     }
1053     return failed.toArray(new HTableDescriptor[failed.size()]);
1054   }
1055 
1056   /*
1057    * Checks whether table exists. If not, throws TableNotFoundException
1058    * @param tableName
1059    */
1060   private void checkTableExistence(TableName tableName) throws IOException {
1061     if (!tableExists(tableName)) {
1062       throw new TableNotFoundException(tableName);
1063     }
1064   }
1065 
1066   /**
1067    * @param tableName name of table to check
1068    * @return true if table is on-line
1069    * @throws IOException if a remote or network exception occurs
1070    */
1071   public boolean isTableEnabled(TableName tableName) throws IOException {
1072     checkTableExistence(tableName);
1073     return connection.isTableEnabled(tableName);
1074   }
1075 
1076   public boolean isTableEnabled(byte[] tableName) throws IOException {
1077     return isTableEnabled(TableName.valueOf(tableName));
1078   }
1079 
1080   public boolean isTableEnabled(String tableName) throws IOException {
1081     return isTableEnabled(TableName.valueOf(tableName));
1082   }
1083 
1084 
1085 
1086   /**
1087    * @param tableName name of table to check
1088    * @return true if table is off-line
1089    * @throws IOException if a remote or network exception occurs
1090    */
1091   public boolean isTableDisabled(TableName tableName) throws IOException {
1092     checkTableExistence(tableName);
1093     return connection.isTableDisabled(tableName);
1094   }
1095 
1096   public boolean isTableDisabled(byte[] tableName) throws IOException {
1097     return isTableDisabled(TableName.valueOf(tableName));
1098   }
1099 
1100   public boolean isTableDisabled(String tableName) throws IOException {
1101     return isTableDisabled(TableName.valueOf(tableName));
1102   }
1103 
1104   /**
1105    * @param tableName name of table to check
1106    * @return true if all regions of the table are available
1107    * @throws IOException if a remote or network exception occurs
1108    */
1109   public boolean isTableAvailable(TableName tableName) throws IOException {
1110     return connection.isTableAvailable(tableName);
1111   }
1112 
1113   public boolean isTableAvailable(byte[] tableName) throws IOException {
1114     return isTableAvailable(TableName.valueOf(tableName));
1115   }
1116 
1117   public boolean isTableAvailable(String tableName) throws IOException {
1118     return isTableAvailable(TableName.valueOf(tableName));
1119   }
1120 
1121   /**
1122    * Use this api to check if the table has been created with the specified number of
1123    * splitkeys which was used while creating the given table.
1124    * Note : If this api is used after a table's region gets splitted, the api may return
1125    * false.
1126    * @param tableName
1127    *          name of table to check
1128    * @param splitKeys
1129    *          keys to check if the table has been created with all split keys
1130    * @throws IOException
1131    *           if a remote or network excpetion occurs
1132    */
1133   public boolean isTableAvailable(TableName tableName,
1134                                   byte[][] splitKeys) throws IOException {
1135     return connection.isTableAvailable(tableName, splitKeys);
1136   }
1137 
1138   public boolean isTableAvailable(byte[] tableName,
1139                                   byte[][] splitKeys) throws IOException {
1140     return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1141   }
1142 
1143   public boolean isTableAvailable(String tableName,
1144                                   byte[][] splitKeys) throws IOException {
1145     return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1146   }
1147 
1148   /**
1149    * Get the status of alter command - indicates how many regions have received
1150    * the updated schema Asynchronous operation.
1151    *
1152    * @param tableName TableName instance
1153    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1154    *         regions that are yet to be updated Pair.getSecond() is the total number
1155    *         of regions of the table
1156    * @throws IOException
1157    *           if a remote or network exception occurs
1158    */
1159   public Pair<Integer, Integer> getAlterStatus(final TableName tableName)
1160   throws IOException {
1161     return executeCallable(new MasterCallable<Pair<Integer, Integer>>(getConnection()) {
1162       @Override
1163       public Pair<Integer, Integer> call() throws ServiceException {
1164         GetSchemaAlterStatusRequest req = RequestConverter
1165             .buildGetSchemaAlterStatusRequest(tableName);
1166         GetSchemaAlterStatusResponse ret = master.getSchemaAlterStatus(null, req);
1167         Pair<Integer, Integer> pair = new Pair<Integer, Integer>(Integer.valueOf(ret
1168             .getYetToUpdateRegions()), Integer.valueOf(ret.getTotalRegions()));
1169         return pair;
1170       }
1171     });
1172   }
1173 
1174   /**
1175    * Get the status of alter command - indicates how many regions have received
1176    * the updated schema Asynchronous operation.
1177    *
1178    * @param tableName
1179    *          name of the table to get the status of
1180    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1181    *         regions that are yet to be updated Pair.getSecond() is the total number
1182    *         of regions of the table
1183    * @throws IOException
1184    *           if a remote or network exception occurs
1185    */
1186   public Pair<Integer, Integer> getAlterStatus(final byte[] tableName)
1187    throws IOException {
1188     return getAlterStatus(TableName.valueOf(tableName));
1189   }
1190 
1191   /**
1192    * Add a column to an existing table.
1193    * Asynchronous operation.
1194    *
1195    * @param tableName name of the table to add column to
1196    * @param column column descriptor of column to be added
1197    * @throws IOException if a remote or network exception occurs
1198    */
1199   public void addColumn(final byte[] tableName, HColumnDescriptor column)
1200   throws IOException {
1201     addColumn(TableName.valueOf(tableName), column);
1202   }
1203 
1204 
1205   /**
1206    * Add a column to an existing table.
1207    * Asynchronous operation.
1208    *
1209    * @param tableName name of the table to add column to
1210    * @param column column descriptor of column to be added
1211    * @throws IOException if a remote or network exception occurs
1212    */
1213   public void addColumn(final String tableName, HColumnDescriptor column)
1214   throws IOException {
1215     addColumn(TableName.valueOf(tableName), column);
1216   }
1217 
1218   /**
1219    * Add a column to an existing table.
1220    * Asynchronous operation.
1221    *
1222    * @param tableName name of the table to add column to
1223    * @param column column descriptor of column to be added
1224    * @throws IOException if a remote or network exception occurs
1225    */
1226   public void addColumn(final TableName tableName, final HColumnDescriptor column)
1227   throws IOException {
1228     executeCallable(new MasterCallable<Void>(getConnection()) {
1229       @Override
1230       public Void call() throws ServiceException {
1231         AddColumnRequest req = RequestConverter.buildAddColumnRequest(tableName, column);
1232         master.addColumn(null,req);
1233         return null;
1234       }
1235     });
1236   }
1237 
1238   /**
1239    * Delete a column from a table.
1240    * Asynchronous operation.
1241    *
1242    * @param tableName name of table
1243    * @param columnName name of column to be deleted
1244    * @throws IOException if a remote or network exception occurs
1245    */
1246   public void deleteColumn(final byte[] tableName, final String columnName)
1247   throws IOException {
1248     deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
1249   }
1250 
1251   /**
1252    * Delete a column from a table.
1253    * Asynchronous operation.
1254    *
1255    * @param tableName name of table
1256    * @param columnName name of column to be deleted
1257    * @throws IOException if a remote or network exception occurs
1258    */
1259   public void deleteColumn(final String tableName, final String columnName)
1260   throws IOException {
1261     deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
1262   }
1263 
1264   /**
1265    * Delete a column from a table.
1266    * Asynchronous operation.
1267    *
1268    * @param tableName name of table
1269    * @param columnName name of column to be deleted
1270    * @throws IOException if a remote or network exception occurs
1271    */
1272   public void deleteColumn(final TableName tableName, final byte [] columnName)
1273   throws IOException {
1274     executeCallable(new MasterCallable<Void>(getConnection()) {
1275       @Override
1276       public Void call() throws ServiceException {
1277         DeleteColumnRequest req = RequestConverter.buildDeleteColumnRequest(tableName, columnName);
1278         master.deleteColumn(null,req);
1279         return null;
1280       }
1281     });
1282   }
1283 
1284   /**
1285    * Modify an existing column family on a table.
1286    * Asynchronous operation.
1287    *
1288    * @param tableName name of table
1289    * @param descriptor new column descriptor to use
1290    * @throws IOException if a remote or network exception occurs
1291    */
1292   public void modifyColumn(final String tableName, HColumnDescriptor descriptor)
1293   throws IOException {
1294     modifyColumn(TableName.valueOf(tableName), descriptor);
1295   }
1296 
1297   /**
1298    * Modify an existing column family on a table.
1299    * Asynchronous operation.
1300    *
1301    * @param tableName name of table
1302    * @param descriptor new column descriptor to use
1303    * @throws IOException if a remote or network exception occurs
1304    */
1305   public void modifyColumn(final byte[] tableName, HColumnDescriptor descriptor)
1306   throws IOException {
1307     modifyColumn(TableName.valueOf(tableName), descriptor);
1308   }
1309 
1310 
1311 
1312   /**
1313    * Modify an existing column family on a table.
1314    * Asynchronous operation.
1315    *
1316    * @param tableName name of table
1317    * @param descriptor new column descriptor to use
1318    * @throws IOException if a remote or network exception occurs
1319    */
1320   public void modifyColumn(final TableName tableName, final HColumnDescriptor descriptor)
1321   throws IOException {
1322     executeCallable(new MasterCallable<Void>(getConnection()) {
1323       @Override
1324       public Void call() throws ServiceException {
1325         ModifyColumnRequest req = RequestConverter.buildModifyColumnRequest(tableName, descriptor);
1326         master.modifyColumn(null,req);
1327         return null;
1328       }
1329     });
1330   }
1331 
1332   /**
1333    * Close a region. For expert-admins.  Runs close on the regionserver.  The
1334    * master will not be informed of the close.
1335    * @param regionname region name to close
1336    * @param serverName If supplied, we'll use this location rather than
1337    * the one currently in <code>hbase:meta</code>
1338    * @throws IOException if a remote or network exception occurs
1339    */
1340   public void closeRegion(final String regionname, final String serverName)
1341   throws IOException {
1342     closeRegion(Bytes.toBytes(regionname), serverName);
1343   }
1344 
1345   /**
1346    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1347    * master will not be informed of the close.
1348    * @param regionname region name to close
1349    * @param serverName The servername of the regionserver.  If passed null we
1350    * will use servername found in the hbase:meta table. A server name
1351    * is made of host, port and startcode.  Here is an example:
1352    * <code> host187.example.com,60020,1289493121758</code>
1353    * @throws IOException if a remote or network exception occurs
1354    */
1355   public void closeRegion(final byte [] regionname, final String serverName)
1356   throws IOException {
1357     CatalogTracker ct = getCatalogTracker();
1358     try {
1359       if (serverName != null) {
1360         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1361         if (pair == null || pair.getFirst() == null) {
1362           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1363         } else {
1364           closeRegion(ServerName.valueOf(serverName), pair.getFirst());
1365         }
1366       } else {
1367         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1368         if (pair == null) {
1369           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1370         } else if (pair.getSecond() == null) {
1371           throw new NoServerForRegionException(Bytes.toStringBinary(regionname));
1372         } else {
1373           closeRegion(pair.getSecond(), pair.getFirst());
1374         }
1375       }
1376     } finally {
1377       cleanupCatalogTracker(ct);
1378     }
1379   }
1380 
1381   /**
1382    * For expert-admins. Runs close on the regionserver. Closes a region based on
1383    * the encoded region name. The region server name is mandatory. If the
1384    * servername is provided then based on the online regions in the specified
1385    * regionserver the specified region will be closed. The master will not be
1386    * informed of the close. Note that the regionname is the encoded regionname.
1387    *
1388    * @param encodedRegionName
1389    *          The encoded region name; i.e. the hash that makes up the region
1390    *          name suffix: e.g. if regionname is
1391    *          <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>
1392    *          , then the encoded region name is:
1393    *          <code>527db22f95c8a9e0116f0cc13c680396</code>.
1394    * @param serverName
1395    *          The servername of the regionserver. A server name is made of host,
1396    *          port and startcode. This is mandatory. Here is an example:
1397    *          <code> host187.example.com,60020,1289493121758</code>
1398    * @return true if the region was closed, false if not.
1399    * @throws IOException
1400    *           if a remote or network exception occurs
1401    */
1402   public boolean closeRegionWithEncodedRegionName(final String encodedRegionName,
1403       final String serverName) throws IOException {
1404     if (null == serverName || ("").equals(serverName.trim())) {
1405       throw new IllegalArgumentException(
1406           "The servername cannot be null or empty.");
1407     }
1408     ServerName sn = ServerName.valueOf(serverName);
1409     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1410     // Close the region without updating zk state.
1411     CloseRegionRequest request =
1412       RequestConverter.buildCloseRegionRequest(sn, encodedRegionName, false);
1413     try {
1414       CloseRegionResponse response = admin.closeRegion(null, request);
1415       boolean isRegionClosed = response.getClosed();
1416       if (false == isRegionClosed) {
1417         LOG.error("Not able to close the region " + encodedRegionName + ".");
1418       }
1419       return isRegionClosed;
1420     } catch (ServiceException se) {
1421       throw ProtobufUtil.getRemoteException(se);
1422     }
1423   }
1424 
1425   /**
1426    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1427    * master will not be informed of the close.
1428    * @param sn
1429    * @param hri
1430    * @throws IOException
1431    */
1432   public void closeRegion(final ServerName sn, final HRegionInfo hri)
1433   throws IOException {
1434     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1435     // Close the region without updating zk state.
1436     ProtobufUtil.closeRegion(admin, sn, hri.getRegionName(), false);
1437   }
1438 
1439   /**
1440    * Get all the online regions on a region server.
1441    */
1442   public List<HRegionInfo> getOnlineRegions(
1443       final ServerName sn) throws IOException {
1444     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1445     return ProtobufUtil.getOnlineRegions(admin);
1446   }
1447 
1448   /**
1449    * Flush a table or an individual region.
1450    * Synchronous operation.
1451    *
1452    * @param tableNameOrRegionName table or region to flush
1453    * @throws IOException if a remote or network exception occurs
1454    * @throws InterruptedException
1455    */
1456   public void flush(final String tableNameOrRegionName)
1457   throws IOException, InterruptedException {
1458     flush(Bytes.toBytes(tableNameOrRegionName));
1459   }
1460 
1461   /**
1462    * Flush a table or an individual region.
1463    * Synchronous operation.
1464    *
1465    * @param tableNameOrRegionName table or region to flush
1466    * @throws IOException if a remote or network exception occurs
1467    * @throws InterruptedException
1468    */
1469   public void flush(final byte[] tableNameOrRegionName)
1470   throws IOException, InterruptedException {
1471     CatalogTracker ct = getCatalogTracker();
1472     try {
1473       Pair<HRegionInfo, ServerName> regionServerPair
1474         = getRegion(tableNameOrRegionName, ct);
1475       if (regionServerPair != null) {
1476         if (regionServerPair.getSecond() == null) {
1477           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1478         } else {
1479           flush(regionServerPair.getSecond(), regionServerPair.getFirst());
1480         }
1481       } else {
1482         final TableName tableName = checkTableExists(
1483             TableName.valueOf(tableNameOrRegionName), ct);
1484         List<Pair<HRegionInfo, ServerName>> pairs =
1485           MetaReader.getTableRegionsAndLocations(ct,
1486               tableName);
1487         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1488           if (pair.getFirst().isOffline()) continue;
1489           if (pair.getSecond() == null) continue;
1490           try {
1491             flush(pair.getSecond(), pair.getFirst());
1492           } catch (NotServingRegionException e) {
1493             if (LOG.isDebugEnabled()) {
1494               LOG.debug("Trying to flush " + pair.getFirst() + ": " +
1495                 StringUtils.stringifyException(e));
1496             }
1497           }
1498         }
1499       }
1500     } finally {
1501       cleanupCatalogTracker(ct);
1502     }
1503   }
1504 
1505   private void flush(final ServerName sn, final HRegionInfo hri)
1506   throws IOException {
1507     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1508     FlushRegionRequest request =
1509       RequestConverter.buildFlushRegionRequest(hri.getRegionName());
1510     try {
1511       admin.flushRegion(null, request);
1512     } catch (ServiceException se) {
1513       throw ProtobufUtil.getRemoteException(se);
1514     }
1515   }
1516 
1517   /**
1518    * Compact a table or an individual region.
1519    * Asynchronous operation.
1520    *
1521    * @param tableNameOrRegionName table or region to compact
1522    * @throws IOException if a remote or network exception occurs
1523    * @throws InterruptedException
1524    */
1525   public void compact(final String tableNameOrRegionName)
1526   throws IOException, InterruptedException {
1527     compact(Bytes.toBytes(tableNameOrRegionName));
1528   }
1529 
1530   /**
1531    * Compact a table or an individual region.
1532    * Asynchronous operation.
1533    *
1534    * @param tableNameOrRegionName table or region to compact
1535    * @throws IOException if a remote or network exception occurs
1536    * @throws InterruptedException
1537    */
1538   public void compact(final byte[] tableNameOrRegionName)
1539   throws IOException, InterruptedException {
1540     compact(tableNameOrRegionName, null, false);
1541   }
1542 
1543   /**
1544    * Compact a column family within a table or region.
1545    * Asynchronous operation.
1546    *
1547    * @param tableOrRegionName table or region to compact
1548    * @param columnFamily column family within a table or region
1549    * @throws IOException if a remote or network exception occurs
1550    * @throws InterruptedException
1551    */
1552   public void compact(String tableOrRegionName, String columnFamily)
1553     throws IOException,  InterruptedException {
1554     compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily));
1555   }
1556 
1557   /**
1558    * Compact a column family within a table or region.
1559    * Asynchronous operation.
1560    *
1561    * @param tableNameOrRegionName table or region to compact
1562    * @param columnFamily column family within a table or region
1563    * @throws IOException if a remote or network exception occurs
1564    * @throws InterruptedException
1565    */
1566   public void compact(final byte[] tableNameOrRegionName, final byte[] columnFamily)
1567   throws IOException, InterruptedException {
1568     compact(tableNameOrRegionName, columnFamily, false);
1569   }
1570 
1571   /**
1572    * Major compact a table or an individual region.
1573    * Asynchronous operation.
1574    *
1575    * @param tableNameOrRegionName table or region to major compact
1576    * @throws IOException if a remote or network exception occurs
1577    * @throws InterruptedException
1578    */
1579   public void majorCompact(final String tableNameOrRegionName)
1580   throws IOException, InterruptedException {
1581     majorCompact(Bytes.toBytes(tableNameOrRegionName));
1582   }
1583 
1584   /**
1585    * Major compact a table or an individual region.
1586    * Asynchronous operation.
1587    *
1588    * @param tableNameOrRegionName table or region to major compact
1589    * @throws IOException if a remote or network exception occurs
1590    * @throws InterruptedException
1591    */
1592   public void majorCompact(final byte[] tableNameOrRegionName)
1593   throws IOException, InterruptedException {
1594     compact(tableNameOrRegionName, null, true);
1595   }
1596 
1597   /**
1598    * Major compact a column family within a table or region.
1599    * Asynchronous operation.
1600    *
1601    * @param tableNameOrRegionName table or region to major compact
1602    * @param columnFamily column family within a table or region
1603    * @throws IOException if a remote or network exception occurs
1604    * @throws InterruptedException
1605    */
1606   public void majorCompact(final String tableNameOrRegionName,
1607     final String columnFamily) throws IOException, InterruptedException {
1608     majorCompact(Bytes.toBytes(tableNameOrRegionName),
1609       Bytes.toBytes(columnFamily));
1610   }
1611 
1612   /**
1613    * Major compact a column family within a table or region.
1614    * Asynchronous operation.
1615    *
1616    * @param tableNameOrRegionName table or region to major compact
1617    * @param columnFamily column family within a table or region
1618    * @throws IOException if a remote or network exception occurs
1619    * @throws InterruptedException
1620    */
1621   public void majorCompact(final byte[] tableNameOrRegionName,
1622     final byte[] columnFamily) throws IOException, InterruptedException {
1623     compact(tableNameOrRegionName, columnFamily, true);
1624   }
1625 
1626   /**
1627    * Compact a table or an individual region.
1628    * Asynchronous operation.
1629    *
1630    * @param tableNameOrRegionName table or region to compact
1631    * @param columnFamily column family within a table or region
1632    * @param major True if we are to do a major compaction.
1633    * @throws IOException if a remote or network exception occurs
1634    * @throws InterruptedException
1635    */
1636   private void compact(final byte[] tableNameOrRegionName,
1637     final byte[] columnFamily,final boolean major)
1638   throws IOException, InterruptedException {
1639     CatalogTracker ct = getCatalogTracker();
1640     try {
1641       Pair<HRegionInfo, ServerName> regionServerPair
1642         = getRegion(tableNameOrRegionName, ct);
1643       if (regionServerPair != null) {
1644         if (regionServerPair.getSecond() == null) {
1645           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1646         } else {
1647           compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
1648         }
1649       } else {
1650         final TableName tableName =
1651             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
1652         List<Pair<HRegionInfo, ServerName>> pairs =
1653           MetaReader.getTableRegionsAndLocations(ct,
1654               tableName);
1655         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1656           if (pair.getFirst().isOffline()) continue;
1657           if (pair.getSecond() == null) continue;
1658           try {
1659             compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
1660           } catch (NotServingRegionException e) {
1661             if (LOG.isDebugEnabled()) {
1662               LOG.debug("Trying to" + (major ? " major" : "") + " compact " +
1663                 pair.getFirst() + ": " +
1664                 StringUtils.stringifyException(e));
1665             }
1666           }
1667         }
1668       }
1669     } finally {
1670       cleanupCatalogTracker(ct);
1671     }
1672   }
1673 
1674   private void compact(final ServerName sn, final HRegionInfo hri,
1675       final boolean major, final byte [] family)
1676   throws IOException {
1677     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1678     CompactRegionRequest request =
1679       RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major, family);
1680     try {
1681       admin.compactRegion(null, request);
1682     } catch (ServiceException se) {
1683       throw ProtobufUtil.getRemoteException(se);
1684     }
1685   }
1686 
1687   /**
1688    * Compact all regions on the region server
1689    * @param regionserver the region server name
1690    * @param major if it's major compaction
1691    * @throws IOException
1692    * @throws InterruptedException
1693    */
1694   public void compactRegionServer(final ServerName sn, boolean major)
1695       throws IOException, InterruptedException {
1696     for (HRegionInfo region : getOnlineRegions(sn)) {
1697       compact(sn, region, major, null);
1698     }
1699   }
1700 
1701   /**
1702    * Move the region <code>r</code> to <code>dest</code>.
1703    * @param encodedRegionName The encoded region name; i.e. the hash that makes
1704    * up the region name suffix: e.g. if regionname is
1705    * <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>,
1706    * then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>.
1707    * @param destServerName The servername of the destination regionserver.  If
1708    * passed the empty byte array we'll assign to a random server.  A server name
1709    * is made of host, port and startcode.  Here is an example:
1710    * <code> host187.example.com,60020,1289493121758</code>
1711    * @throws UnknownRegionException Thrown if we can't find a region named
1712    * <code>encodedRegionName</code>
1713    * @throws ZooKeeperConnectionException
1714    * @throws MasterNotRunningException
1715    */
1716   public void move(final byte [] encodedRegionName, final byte [] destServerName)
1717   throws HBaseIOException, MasterNotRunningException, ZooKeeperConnectionException {
1718     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1719     try {
1720       MoveRegionRequest request =
1721         RequestConverter.buildMoveRegionRequest(encodedRegionName, destServerName);
1722       stub.moveRegion(null,request);
1723     } catch (ServiceException se) {
1724       IOException ioe = ProtobufUtil.getRemoteException(se);
1725       if (ioe instanceof HBaseIOException) {
1726         throw (HBaseIOException)ioe;
1727       }
1728       LOG.error("Unexpected exception: " + se + " from calling HMaster.moveRegion");
1729     } catch (DeserializationException de) {
1730       LOG.error("Could not parse destination server name: " + de);
1731     } finally {
1732       stub.close();
1733     }
1734   }
1735 
1736   /**
1737    * @param regionName
1738    *          Region name to assign.
1739    * @throws MasterNotRunningException
1740    * @throws ZooKeeperConnectionException
1741    * @throws IOException
1742    */
1743   public void assign(final byte[] regionName) throws MasterNotRunningException,
1744       ZooKeeperConnectionException, IOException {
1745     final byte[] toBeAssigned = getRegionName(regionName);
1746     executeCallable(new MasterCallable<Void>(getConnection()) {
1747       @Override
1748       public Void call() throws ServiceException {
1749         AssignRegionRequest request =
1750           RequestConverter.buildAssignRegionRequest(toBeAssigned);
1751         master.assignRegion(null,request);
1752         return null;
1753       }
1754     });
1755   }
1756 
1757   /**
1758    * Unassign a region from current hosting regionserver.  Region will then be
1759    * assigned to a regionserver chosen at random.  Region could be reassigned
1760    * back to the same server.  Use {@link #move(byte[], byte[])} if you want
1761    * to control the region movement.
1762    * @param regionName Region to unassign. Will clear any existing RegionPlan
1763    * if one found.
1764    * @param force If true, force unassign (Will remove region from
1765    * regions-in-transition too if present. If results in double assignment
1766    * use hbck -fix to resolve. To be used by experts).
1767    * @throws MasterNotRunningException
1768    * @throws ZooKeeperConnectionException
1769    * @throws IOException
1770    */
1771   public void unassign(final byte [] regionName, final boolean force)
1772   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
1773     final byte[] toBeUnassigned = getRegionName(regionName);
1774     executeCallable(new MasterCallable<Void>(getConnection()) {
1775       @Override
1776       public Void call() throws ServiceException {
1777         UnassignRegionRequest request =
1778           RequestConverter.buildUnassignRegionRequest(toBeUnassigned, force);
1779         master.unassignRegion(null,request);
1780         return null;
1781       }
1782     });
1783   }
1784 
1785   /**
1786    * Offline specified region from master's in-memory state. It will not attempt to reassign the
1787    * region as in unassign. This API can be used when a region not served by any region server and
1788    * still online as per Master's in memory state. If this API is incorrectly used on active region
1789    * then master will loose track of that region.
1790    *
1791    * This is a special method that should be used by experts or hbck.
1792    *
1793    * @param regionName
1794    *          Region to offline.
1795    * @throws IOException
1796    */
1797   public void offline(final byte [] regionName)
1798   throws IOException {
1799     MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
1800     try {
1801       master.offlineRegion(null,RequestConverter.buildOfflineRegionRequest(regionName));
1802     } catch (ServiceException se) {
1803       throw ProtobufUtil.getRemoteException(se);
1804     } finally {
1805       master.close();
1806     }
1807   }
1808 
1809   /**
1810    * Turn the load balancer on or off.
1811    * @param on If true, enable balancer. If false, disable balancer.
1812    * @param synchronous If true, it waits until current balance() call, if outstanding, to return.
1813    * @return Previous balancer value
1814    */
1815   public boolean setBalancerRunning(final boolean on, final boolean synchronous)
1816   throws MasterNotRunningException, ZooKeeperConnectionException {
1817     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1818     try {
1819       SetBalancerRunningRequest req =
1820         RequestConverter.buildSetBalancerRunningRequest(on, synchronous);
1821       return stub.setBalancerRunning(null, req).getPrevBalanceValue();
1822     } catch (ServiceException se) {
1823       IOException ioe = ProtobufUtil.getRemoteException(se);
1824       if (ioe instanceof MasterNotRunningException) {
1825         throw (MasterNotRunningException)ioe;
1826       }
1827       if (ioe instanceof ZooKeeperConnectionException) {
1828         throw (ZooKeeperConnectionException)ioe;
1829       }
1830 
1831       // Throwing MasterNotRunningException even though not really valid in order to not
1832       // break interface by adding additional exception type.
1833       throw new MasterNotRunningException("Unexpected exception when calling balanceSwitch",se);
1834     } finally {
1835       stub.close();
1836     }
1837   }
1838 
1839   /**
1840    * Invoke the balancer.  Will run the balancer and if regions to move, it will
1841    * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
1842    * logs.
1843    * @return True if balancer ran, false otherwise.
1844    */
1845   public boolean balancer()
1846   throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException {
1847     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1848     try {
1849       return stub.balance(null,RequestConverter.buildBalanceRequest()).getBalancerRan();
1850     } finally {
1851       stub.close();
1852     }
1853   }
1854 
1855   /**
1856    * Query the state of the balancer from the Master. It's not a guarantee that the balancer is
1857    * actually running this very moment, but that it will run.
1858    *
1859    * @return True if the balancer is enabled, false otherwise.
1860    */
1861   public boolean isBalancerEnabled()
1862       throws ServiceException, MasterNotRunningException {
1863     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1864     try {
1865       return stub.isBalancerEnabled(null, RequestConverter.buildIsBalancerEnabledRequest())
1866         .getEnabled();
1867     } finally {
1868       stub.close();
1869     }
1870   }
1871 
1872   /**
1873    * Enable/Disable the catalog janitor
1874    * @param enable if true enables the catalog janitor
1875    * @return the previous state
1876    * @throws ServiceException
1877    * @throws MasterNotRunningException
1878    */
1879   public boolean enableCatalogJanitor(boolean enable)
1880       throws ServiceException, MasterNotRunningException {
1881     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1882     try {
1883       return stub.enableCatalogJanitor(null,
1884           RequestConverter.buildEnableCatalogJanitorRequest(enable)).getPrevValue();
1885     } finally {
1886       stub.close();
1887     }
1888   }
1889 
1890   /**
1891    * Ask for a scan of the catalog table
1892    * @return the number of entries cleaned
1893    * @throws ServiceException
1894    * @throws MasterNotRunningException
1895    */
1896   public int runCatalogScan() throws ServiceException, MasterNotRunningException {
1897     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1898     try {
1899       return stub.runCatalogScan(null,
1900           RequestConverter.buildCatalogScanRequest()).getScanResult();
1901     } finally {
1902       stub.close();
1903     }
1904   }
1905 
1906   /**
1907    * Query on the catalog janitor state (Enabled/Disabled?)
1908    * @throws ServiceException
1909    * @throws org.apache.hadoop.hbase.MasterNotRunningException
1910    */
1911   public boolean isCatalogJanitorEnabled() throws ServiceException, MasterNotRunningException {
1912     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1913     try {
1914       return stub.isCatalogJanitorEnabled(null,
1915           RequestConverter.buildIsCatalogJanitorEnabledRequest()).getValue();
1916     } finally {
1917       stub.close();
1918     }
1919   }
1920 
1921   /**
1922    * Merge two regions. Asynchronous operation.
1923    * @param encodedNameOfRegionA encoded name of region a
1924    * @param encodedNameOfRegionB encoded name of region b
1925    * @param forcible true if do a compulsory merge, otherwise we will only merge
1926    *          two adjacent regions
1927    * @throws IOException
1928    */
1929   public void mergeRegions(final byte[] encodedNameOfRegionA,
1930       final byte[] encodedNameOfRegionB, final boolean forcible)
1931       throws IOException {
1932     MasterKeepAliveConnection master = connection
1933         .getKeepAliveMasterService();
1934     try {
1935       DispatchMergingRegionsRequest request = RequestConverter
1936           .buildDispatchMergingRegionsRequest(encodedNameOfRegionA,
1937               encodedNameOfRegionB, forcible);
1938       master.dispatchMergingRegions(null, request);
1939     } catch (ServiceException se) {
1940       IOException ioe = ProtobufUtil.getRemoteException(se);
1941       if (ioe instanceof UnknownRegionException) {
1942         throw (UnknownRegionException) ioe;
1943       }
1944       if (ioe instanceof MergeRegionException) {
1945         throw (MergeRegionException) ioe;
1946       }
1947       LOG.error("Unexpected exception: " + se
1948           + " from calling HMaster.dispatchMergingRegions");
1949     } catch (DeserializationException de) {
1950       LOG.error("Could not parse destination server name: " + de);
1951     } finally {
1952       master.close();
1953     }
1954   }
1955 
1956   /**
1957    * Split a table or an individual region.
1958    * Asynchronous operation.
1959    *
1960    * @param tableNameOrRegionName table or region to split
1961    * @throws IOException if a remote or network exception occurs
1962    * @throws InterruptedException
1963    */
1964   public void split(final String tableNameOrRegionName)
1965   throws IOException, InterruptedException {
1966     split(Bytes.toBytes(tableNameOrRegionName));
1967   }
1968 
1969   /**
1970    * Split a table or an individual region.  Implicitly finds an optimal split
1971    * point.  Asynchronous operation.
1972    *
1973    * @param tableNameOrRegionName table to region to split
1974    * @throws IOException if a remote or network exception occurs
1975    * @throws InterruptedException
1976    */
1977   public void split(final byte[] tableNameOrRegionName)
1978   throws IOException, InterruptedException {
1979     split(tableNameOrRegionName, null);
1980   }
1981 
1982   public void split(final String tableNameOrRegionName,
1983     final String splitPoint) throws IOException, InterruptedException {
1984     split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint));
1985   }
1986 
1987   /**
1988    * Split a table or an individual region.
1989    * Asynchronous operation.
1990    *
1991    * @param tableNameOrRegionName table to region to split
1992    * @param splitPoint the explicit position to split on
1993    * @throws IOException if a remote or network exception occurs
1994    * @throws InterruptedException interrupt exception occurred
1995    */
1996   public void split(final byte[] tableNameOrRegionName,
1997       final byte [] splitPoint) throws IOException, InterruptedException {
1998     CatalogTracker ct = getCatalogTracker();
1999     try {
2000       Pair<HRegionInfo, ServerName> regionServerPair
2001         = getRegion(tableNameOrRegionName, ct);
2002       if (regionServerPair != null) {
2003         if (regionServerPair.getSecond() == null) {
2004             throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
2005         } else {
2006           split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
2007         }
2008       } else {
2009         final TableName tableName =
2010             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
2011         List<Pair<HRegionInfo, ServerName>> pairs =
2012           MetaReader.getTableRegionsAndLocations(ct,
2013               tableName);
2014         for (Pair<HRegionInfo, ServerName> pair: pairs) {
2015           // May not be a server for a particular row
2016           if (pair.getSecond() == null) continue;
2017           HRegionInfo r = pair.getFirst();
2018           // check for parents
2019           if (r.isSplitParent()) continue;
2020           // if a split point given, only split that particular region
2021           if (splitPoint != null && !r.containsRow(splitPoint)) continue;
2022           // call out to region server to do split now
2023           split(pair.getSecond(), pair.getFirst(), splitPoint);
2024         }
2025       }
2026     } finally {
2027       cleanupCatalogTracker(ct);
2028     }
2029   }
2030 
2031   private void split(final ServerName sn, final HRegionInfo hri,
2032       byte[] splitPoint) throws IOException {
2033     if (hri.getStartKey() != null && splitPoint != null &&
2034          Bytes.compareTo(hri.getStartKey(), splitPoint) == 0) {
2035        throw new IOException("should not give a splitkey which equals to startkey!");
2036     }
2037     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2038     ProtobufUtil.split(admin, hri, splitPoint);
2039   }
2040 
2041   /**
2042    * Modify an existing table, more IRB friendly version.
2043    * Asynchronous operation.  This means that it may be a while before your
2044    * schema change is updated across all of the table.
2045    *
2046    * @param tableName name of table.
2047    * @param htd modified description of the table
2048    * @throws IOException if a remote or network exception occurs
2049    */
2050   public void modifyTable(final TableName tableName, final HTableDescriptor htd)
2051   throws IOException {
2052     if (!tableName.equals(htd.getTableName())) {
2053       throw new IllegalArgumentException("the specified table name '" + tableName +
2054         "' doesn't match with the HTD one: " + htd.getTableName());
2055     }
2056 
2057     executeCallable(new MasterCallable<Void>(getConnection()) {
2058       @Override
2059       public Void call() throws ServiceException {
2060         ModifyTableRequest request = RequestConverter.buildModifyTableRequest(tableName, htd);
2061         master.modifyTable(null, request);
2062         return null;
2063       }
2064     });
2065   }
2066 
2067   public void modifyTable(final byte[] tableName, final HTableDescriptor htd)
2068   throws IOException {
2069     modifyTable(TableName.valueOf(tableName), htd);
2070   }
2071 
2072   public void modifyTable(final String tableName, final HTableDescriptor htd)
2073   throws IOException {
2074     modifyTable(TableName.valueOf(tableName), htd);
2075   }
2076 
2077   /**
2078    * @param tableNameOrRegionName Name of a table or name of a region.
2079    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
2080    * @return a pair of HRegionInfo and ServerName if <code>tableNameOrRegionName</code> is
2081    *  a verified region name (we call {@link  MetaReader#getRegion( CatalogTracker, byte[])}
2082    *  else null.
2083    * Throw an exception if <code>tableNameOrRegionName</code> is null.
2084    * @throws IOException
2085    */
2086   Pair<HRegionInfo, ServerName> getRegion(final byte[] tableNameOrRegionName,
2087       final CatalogTracker ct) throws IOException {
2088     if (tableNameOrRegionName == null) {
2089       throw new IllegalArgumentException("Pass a table name or region name");
2090     }
2091     Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, tableNameOrRegionName);
2092     if (pair == null) {
2093       final AtomicReference<Pair<HRegionInfo, ServerName>> result =
2094         new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
2095       final String encodedName = Bytes.toString(tableNameOrRegionName);
2096       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2097         @Override
2098         public boolean processRow(Result data) throws IOException {
2099           HRegionInfo info = HRegionInfo.getHRegionInfo(data);
2100           if (info == null) {
2101             LOG.warn("No serialized HRegionInfo in " + data);
2102             return true;
2103           }
2104           if (!encodedName.equals(info.getEncodedName())) return true;
2105           ServerName sn = HRegionInfo.getServerName(data);
2106           result.set(new Pair<HRegionInfo, ServerName>(info, sn));
2107           return false; // found the region, stop
2108         }
2109       };
2110 
2111       MetaScanner.metaScan(conf, connection, visitor, null);
2112       pair = result.get();
2113     }
2114     return pair;
2115   }
2116 
2117   /**
2118    * If the input is a region name, it is returned as is. If it's an
2119    * encoded region name, the corresponding region is found from meta
2120    * and its region name is returned. If we can't find any region in
2121    * meta matching the input as either region name or encoded region
2122    * name, the input is returned as is. We don't throw unknown
2123    * region exception.
2124    */
2125   private byte[] getRegionName(
2126       final byte[] regionNameOrEncodedRegionName) throws IOException {
2127     if (Bytes.equals(regionNameOrEncodedRegionName,
2128         HRegionInfo.FIRST_META_REGIONINFO.getRegionName())
2129           || Bytes.equals(regionNameOrEncodedRegionName,
2130             HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes())) {
2131       return HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
2132     }
2133     CatalogTracker ct = getCatalogTracker();
2134     byte[] tmp = regionNameOrEncodedRegionName;
2135     try {
2136       Pair<HRegionInfo, ServerName> regionServerPair
2137         = getRegion(regionNameOrEncodedRegionName, ct);
2138       if (regionServerPair != null && regionServerPair.getFirst() != null) {
2139         tmp = regionServerPair.getFirst().getRegionName();
2140       }
2141     } finally {
2142       cleanupCatalogTracker(ct);
2143     }
2144     return tmp;
2145   }
2146 
2147   /**
2148    * Check if table exists or not
2149    * @param tableName Name of a table.
2150    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
2151    * @return tableName instance
2152    * @throws IOException if a remote or network exception occurs.
2153    * @throws TableNotFoundException if table does not exist.
2154    */
2155   //TODO rename this method
2156   private TableName checkTableExists(
2157       final TableName tableName, CatalogTracker ct)
2158       throws IOException {
2159     if (!MetaReader.tableExists(ct, tableName)) {
2160       throw new TableNotFoundException(tableName);
2161     }
2162     return tableName;
2163   }
2164 
2165   /**
2166    * Shuts down the HBase cluster
2167    * @throws IOException if a remote or network exception occurs
2168    */
2169   public synchronized void shutdown() throws IOException {
2170     executeCallable(new MasterCallable<Void>(getConnection()) {
2171       @Override
2172       public Void call() throws ServiceException {
2173         master.shutdown(null,ShutdownRequest.newBuilder().build());
2174         return null;
2175       }
2176     });
2177   }
2178 
2179   /**
2180    * Shuts down the current HBase master only.
2181    * Does not shutdown the cluster.
2182    * @see #shutdown()
2183    * @throws IOException if a remote or network exception occurs
2184    */
2185   public synchronized void stopMaster() throws IOException {
2186     executeCallable(new MasterCallable<Void>(getConnection()) {
2187       @Override
2188       public Void call() throws ServiceException {
2189         master.stopMaster(null,StopMasterRequest.newBuilder().build());
2190         return null;
2191       }
2192     });
2193   }
2194 
2195   /**
2196    * Stop the designated regionserver
2197    * @param hostnamePort Hostname and port delimited by a <code>:</code> as in
2198    * <code>example.org:1234</code>
2199    * @throws IOException if a remote or network exception occurs
2200    */
2201   public synchronized void stopRegionServer(final String hostnamePort)
2202   throws IOException {
2203     String hostname = Addressing.parseHostname(hostnamePort);
2204     int port = Addressing.parsePort(hostnamePort);
2205     AdminService.BlockingInterface admin =
2206       this.connection.getAdmin(ServerName.valueOf(hostname, port, 0));
2207     StopServerRequest request = RequestConverter.buildStopServerRequest(
2208       "Called by admin client " + this.connection.toString());
2209     try {
2210       admin.stopServer(null, request);
2211     } catch (ServiceException se) {
2212       throw ProtobufUtil.getRemoteException(se);
2213     }
2214   }
2215 
2216 
2217   /**
2218    * @return cluster status
2219    * @throws IOException if a remote or network exception occurs
2220    */
2221   public ClusterStatus getClusterStatus() throws IOException {
2222     return executeCallable(new MasterCallable<ClusterStatus>(getConnection()) {
2223       @Override
2224       public ClusterStatus call() throws ServiceException {
2225         GetClusterStatusRequest req = RequestConverter.buildGetClusterStatusRequest();
2226         return ClusterStatus.convert(master.getClusterStatus(null,req).getClusterStatus());
2227       }
2228     });
2229   }
2230 
2231   private HRegionLocation getFirstMetaServerForTable(final TableName tableName)
2232   throws IOException {
2233     return connection.locateRegion(TableName.META_TABLE_NAME,
2234       HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
2235   }
2236 
2237   /**
2238    * @return Configuration used by the instance.
2239    */
2240   public Configuration getConfiguration() {
2241     return this.conf;
2242   }
2243 
2244   /**
2245    * Create a new namespace
2246    * @param descriptor descriptor which describes the new namespace
2247    * @throws IOException
2248    */
2249   public void createNamespace(final NamespaceDescriptor descriptor) throws IOException {
2250     executeCallable(new MasterCallable<Void>(getConnection()) {
2251       @Override
2252       public Void call() throws Exception {
2253         master.createNamespace(null,
2254           CreateNamespaceRequest.newBuilder()
2255                 .setNamespaceDescriptor(ProtobufUtil
2256                     .toProtoNamespaceDescriptor(descriptor)).build());
2257         return null;
2258       }
2259     });
2260   }
2261 
2262   /**
2263    * Modify an existing namespace
2264    * @param descriptor descriptor which describes the new namespace
2265    * @throws IOException
2266    */
2267   public void modifyNamespace(final NamespaceDescriptor descriptor) throws IOException {
2268     executeCallable(new MasterCallable<Void>(getConnection()) {
2269       @Override
2270       public Void call() throws Exception {
2271         master.modifyNamespace(null, ModifyNamespaceRequest.newBuilder().
2272           setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(descriptor)).build());
2273         return null;
2274       }
2275     });
2276   }
2277 
2278   /**
2279    * Delete an existing namespace. Only empty namespaces (no tables) can be removed.
2280    * @param name namespace name
2281    * @throws IOException
2282    */
2283   public void deleteNamespace(final String name) throws IOException {
2284     executeCallable(new MasterCallable<Void>(getConnection()) {
2285       @Override
2286       public Void call() throws Exception {
2287         master.deleteNamespace(null, DeleteNamespaceRequest.newBuilder().
2288           setNamespaceName(name).build());
2289         return null;
2290       }
2291     });
2292   }
2293 
2294   /**
2295    * Get a namespace descriptor by name
2296    * @param name name of namespace descriptor
2297    * @return A descriptor
2298    * @throws IOException
2299    */
2300   public NamespaceDescriptor getNamespaceDescriptor(final String name) throws IOException {
2301     return
2302         executeCallable(new MasterCallable<NamespaceDescriptor>(getConnection()) {
2303           @Override
2304           public NamespaceDescriptor call() throws Exception {
2305             return ProtobufUtil.toNamespaceDescriptor(
2306               master.getNamespaceDescriptor(null, GetNamespaceDescriptorRequest.newBuilder().
2307                 setNamespaceName(name).build()).getNamespaceDescriptor());
2308           }
2309         });
2310   }
2311 
2312   /**
2313    * List available namespace descriptors
2314    * @return List of descriptors
2315    * @throws IOException
2316    */
2317   public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException {
2318     return
2319         executeCallable(new MasterCallable<NamespaceDescriptor[]>(getConnection()) {
2320           @Override
2321           public NamespaceDescriptor[] call() throws Exception {
2322             List<HBaseProtos.NamespaceDescriptor> list =
2323               master.listNamespaceDescriptors(null, ListNamespaceDescriptorsRequest.newBuilder().
2324                 build()).getNamespaceDescriptorList();
2325             NamespaceDescriptor[] res = new NamespaceDescriptor[list.size()];
2326             for(int i = 0; i < list.size(); i++) {
2327               res[i] = ProtobufUtil.toNamespaceDescriptor(list.get(i));
2328             }
2329             return res;
2330           }
2331         });
2332   }
2333 
2334   /**
2335    * Get list of table descriptors by namespace
2336    * @param name namespace name
2337    * @return A descriptor
2338    * @throws IOException
2339    */
2340   public HTableDescriptor[] listTableDescriptorsByNamespace(final String name) throws IOException {
2341     return
2342         executeCallable(new MasterCallable<HTableDescriptor[]>(getConnection()) {
2343           @Override
2344           public HTableDescriptor[] call() throws Exception {
2345             List<TableSchema> list =
2346               master.listTableDescriptorsByNamespace(null, ListTableDescriptorsByNamespaceRequest.
2347                 newBuilder().setNamespaceName(name).build()).getTableSchemaList();
2348             HTableDescriptor[] res = new HTableDescriptor[list.size()];
2349             for(int i=0; i < list.size(); i++) {
2350 
2351               res[i] = HTableDescriptor.convert(list.get(i));
2352             }
2353             return res;
2354           }
2355         });
2356   }
2357 
2358   /**
2359    * Get list of table names by namespace
2360    * @param name namespace name
2361    * @return The list of table names in the namespace
2362    * @throws IOException
2363    */
2364   public TableName[] listTableNamesByNamespace(final String name) throws IOException {
2365     return
2366         executeCallable(new MasterCallable<TableName[]>(getConnection()) {
2367           @Override
2368           public TableName[] call() throws Exception {
2369             List<HBaseProtos.TableName> tableNames =
2370               master.listTableNamesByNamespace(null, ListTableNamesByNamespaceRequest.
2371                 newBuilder().setNamespaceName(name).build())
2372                 .getTableNameList();
2373             TableName[] result = new TableName[tableNames.size()];
2374             for (int i = 0; i < tableNames.size(); i++) {
2375               result[i] = ProtobufUtil.toTableName(tableNames.get(i));
2376             }
2377             return result;
2378           }
2379         });
2380   }
2381 
2382   /**
2383    * Check to see if HBase is running. Throw an exception if not.
2384    * We consider that HBase is running if ZooKeeper and Master are running.
2385    *
2386    * @param conf system configuration
2387    * @throws MasterNotRunningException if the master is not running
2388    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
2389    */
2390   public static void checkHBaseAvailable(Configuration conf)
2391     throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException {
2392     Configuration copyOfConf = HBaseConfiguration.create(conf);
2393 
2394     // We set it to make it fail as soon as possible if HBase is not available
2395     copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
2396     copyOfConf.setInt("zookeeper.recovery.retry", 0);
2397 
2398     HConnectionManager.HConnectionImplementation connection
2399       = (HConnectionManager.HConnectionImplementation)
2400       HConnectionManager.getConnection(copyOfConf);
2401 
2402     try {
2403       // Check ZK first.
2404       // If the connection exists, we may have a connection to ZK that does
2405       //  not work anymore
2406       ZooKeeperKeepAliveConnection zkw = null;
2407       try {
2408         zkw = connection.getKeepAliveZooKeeperWatcher();
2409         zkw.getRecoverableZooKeeper().getZooKeeper().exists(
2410           zkw.baseZNode, false);
2411 
2412       } catch (IOException e) {
2413         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
2414       } catch (InterruptedException e) {
2415         throw (InterruptedIOException)
2416             new InterruptedIOException("Can't connect to ZooKeeper").initCause(e);
2417       } catch (KeeperException e) {
2418         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
2419       } finally {
2420         if (zkw != null) {
2421           zkw.close();
2422         }
2423       }
2424 
2425       // Check Master
2426       connection.isMasterRunning();
2427 
2428     } finally {
2429       connection.close();
2430     }
2431   }
2432 
2433   /**
2434    * get the regions of a given table.
2435    *
2436    * @param tableName the name of the table
2437    * @return Ordered list of {@link HRegionInfo}.
2438    * @throws IOException
2439    */
2440   public List<HRegionInfo> getTableRegions(final TableName tableName)
2441   throws IOException {
2442     CatalogTracker ct = getCatalogTracker();
2443     List<HRegionInfo> Regions = null;
2444     try {
2445       Regions = MetaReader.getTableRegions(ct, tableName, true);
2446     } finally {
2447       cleanupCatalogTracker(ct);
2448     }
2449     return Regions;
2450   }
2451 
2452   public List<HRegionInfo> getTableRegions(final byte[] tableName)
2453   throws IOException {
2454     return getTableRegions(TableName.valueOf(tableName));
2455   }
2456 
2457   @Override
2458   public synchronized void close() throws IOException {
2459     if (cleanupConnectionOnClose && this.connection != null && !this.closed) {
2460       this.connection.close();
2461       this.closed = true;
2462     }
2463   }
2464 
2465   /**
2466    * Get tableDescriptors
2467    * @param tableNames List of table names
2468    * @return HTD[] the tableDescriptor
2469    * @throws IOException if a remote or network exception occurs
2470    */
2471   public HTableDescriptor[] getTableDescriptorsByTableName(List<TableName> tableNames)
2472   throws IOException {
2473     return this.connection.getHTableDescriptorsByTableName(tableNames);
2474   }
2475 
2476   /**
2477    * Get tableDescriptors
2478    * @param names List of table names
2479    * @return HTD[] the tableDescriptor
2480    * @throws IOException if a remote or network exception occurs
2481    */
2482   public HTableDescriptor[] getTableDescriptors(List<String> names)
2483   throws IOException {
2484     List<TableName> tableNames = new ArrayList<TableName>(names.size());
2485     for(String name : names) {
2486       tableNames.add(TableName.valueOf(name));
2487     }
2488     return getTableDescriptorsByTableName(tableNames);
2489   }
2490 
2491   /**
2492    * Roll the log writer. That is, start writing log messages to a new file.
2493    *
2494    * @param serverName
2495    *          The servername of the regionserver. A server name is made of host,
2496    *          port and startcode. This is mandatory. Here is an example:
2497    *          <code> host187.example.com,60020,1289493121758</code>
2498    * @return If lots of logs, flush the returned regions so next time through
2499    * we can clean logs. Returns null if nothing to flush.  Names are actual
2500    * region names as returned by {@link HRegionInfo#getEncodedName()}
2501    * @throws IOException if a remote or network exception occurs
2502    * @throws FailedLogCloseException
2503    */
2504  public synchronized  byte[][] rollHLogWriter(String serverName)
2505       throws IOException, FailedLogCloseException {
2506     ServerName sn = ServerName.valueOf(serverName);
2507     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2508     RollWALWriterRequest request = RequestConverter.buildRollWALWriterRequest();
2509     try {
2510       RollWALWriterResponse response = admin.rollWALWriter(null, request);
2511       int regionCount = response.getRegionToFlushCount();
2512       byte[][] regionsToFlush = new byte[regionCount][];
2513       for (int i = 0; i < regionCount; i++) {
2514         ByteString region = response.getRegionToFlush(i);
2515         regionsToFlush[i] = region.toByteArray();
2516       }
2517       return regionsToFlush;
2518     } catch (ServiceException se) {
2519       throw ProtobufUtil.getRemoteException(se);
2520     }
2521   }
2522 
2523   public String[] getMasterCoprocessors() {
2524     try {
2525       return getClusterStatus().getMasterCoprocessors();
2526     } catch (IOException e) {
2527       LOG.error("Could not getClusterStatus()",e);
2528       return null;
2529     }
2530   }
2531 
2532   /**
2533    * Get the current compaction state of a table or region.
2534    * It could be in a major compaction, a minor compaction, both, or none.
2535    *
2536    * @param tableNameOrRegionName table or region to major compact
2537    * @throws IOException if a remote or network exception occurs
2538    * @throws InterruptedException
2539    * @return the current compaction state
2540    */
2541   public CompactionState getCompactionState(final String tableNameOrRegionName)
2542       throws IOException, InterruptedException {
2543     return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
2544   }
2545 
2546   /**
2547    * Get the current compaction state of a table or region.
2548    * It could be in a major compaction, a minor compaction, both, or none.
2549    *
2550    * @param tableNameOrRegionName table or region to major compact
2551    * @throws IOException if a remote or network exception occurs
2552    * @throws InterruptedException
2553    * @return the current compaction state
2554    */
2555   public CompactionState getCompactionState(final byte[] tableNameOrRegionName)
2556       throws IOException, InterruptedException {
2557     CompactionState state = CompactionState.NONE;
2558     CatalogTracker ct = getCatalogTracker();
2559     try {
2560       Pair<HRegionInfo, ServerName> regionServerPair
2561         = getRegion(tableNameOrRegionName, ct);
2562       if (regionServerPair != null) {
2563         if (regionServerPair.getSecond() == null) {
2564           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
2565         } else {
2566           ServerName sn = regionServerPair.getSecond();
2567           AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2568           GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2569             regionServerPair.getFirst().getRegionName(), true);
2570           GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2571           return response.getCompactionState();
2572         }
2573       } else {
2574         final TableName tableName =
2575             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
2576         List<Pair<HRegionInfo, ServerName>> pairs =
2577           MetaReader.getTableRegionsAndLocations(ct, tableName);
2578         for (Pair<HRegionInfo, ServerName> pair: pairs) {
2579           if (pair.getFirst().isOffline()) continue;
2580           if (pair.getSecond() == null) continue;
2581           try {
2582             ServerName sn = pair.getSecond();
2583             AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2584             GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2585               pair.getFirst().getRegionName(), true);
2586             GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2587             switch (response.getCompactionState()) {
2588             case MAJOR_AND_MINOR:
2589               return CompactionState.MAJOR_AND_MINOR;
2590             case MAJOR:
2591               if (state == CompactionState.MINOR) {
2592                 return CompactionState.MAJOR_AND_MINOR;
2593               }
2594               state = CompactionState.MAJOR;
2595               break;
2596             case MINOR:
2597               if (state == CompactionState.MAJOR) {
2598                 return CompactionState.MAJOR_AND_MINOR;
2599               }
2600               state = CompactionState.MINOR;
2601               break;
2602             case NONE:
2603               default: // nothing, continue
2604             }
2605           } catch (NotServingRegionException e) {
2606             if (LOG.isDebugEnabled()) {
2607               LOG.debug("Trying to get compaction state of " +
2608                 pair.getFirst() + ": " +
2609                 StringUtils.stringifyException(e));
2610             }
2611           } catch (RemoteException e) {
2612             if (e.getMessage().indexOf(NotServingRegionException.class.getName()) >= 0) {
2613               if (LOG.isDebugEnabled()) {
2614                 LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": "
2615                     + StringUtils.stringifyException(e));
2616               }
2617             } else {
2618               throw e;
2619             }
2620           }
2621         }
2622       }
2623     } catch (ServiceException se) {
2624       throw ProtobufUtil.getRemoteException(se);
2625     } finally {
2626       cleanupCatalogTracker(ct);
2627     }
2628     return state;
2629   }
2630 
2631   /**
2632    * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be
2633    * taken. If the table is disabled, an offline snapshot is taken.
2634    * <p>
2635    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2636    * snapshot with the same name (even a different type or with different parameters) will fail with
2637    * a {@link SnapshotCreationException} indicating the duplicate naming.
2638    * <p>
2639    * Snapshot names follow the same naming constraints as tables in HBase. See
2640    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2641    * @param snapshotName name of the snapshot to be created
2642    * @param tableName name of the table for which snapshot is created
2643    * @throws IOException if a remote or network exception occurs
2644    * @throws SnapshotCreationException if snapshot creation failed
2645    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2646    */
2647   public void snapshot(final String snapshotName,
2648                        final TableName tableName) throws IOException,
2649       SnapshotCreationException, IllegalArgumentException {
2650     snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
2651   }
2652 
2653   public void snapshot(final String snapshotName,
2654                        final String tableName) throws IOException,
2655       SnapshotCreationException, IllegalArgumentException {
2656     snapshot(snapshotName, TableName.valueOf(tableName),
2657         SnapshotDescription.Type.FLUSH);
2658   }
2659 
2660   /**
2661    * Create snapshot for the given table of given flush type.
2662    * <p>
2663    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2664    * snapshot with the same name (even a different type or with different parameters) will fail with
2665    * a {@link SnapshotCreationException} indicating the duplicate naming.
2666    * <p>
2667    * Snapshot names follow the same naming constraints as tables in HBase.
2668    * @param snapshotName name of the snapshot to be created
2669    * @param tableName name of the table for which snapshot is created
2670    * @param flushType if the snapshot should be taken without flush memstore first
2671    * @throws IOException if a remote or network exception occurs
2672    * @throws SnapshotCreationException if snapshot creation failed
2673    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2674    */
2675    public void snapshot(final byte[] snapshotName, final byte[] tableName,
2676                        final SnapshotDescription.Type flushType) throws
2677       IOException, SnapshotCreationException, IllegalArgumentException {
2678       snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType);
2679   }
2680   /**
2681    public void snapshot(final String snapshotName,
2682     * Create a timestamp consistent snapshot for the given table.
2683                         final byte[] tableName) throws IOException,
2684     * <p>
2685     * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2686     * snapshot with the same name (even a different type or with different parameters) will fail with
2687     * a {@link SnapshotCreationException} indicating the duplicate naming.
2688     * <p>
2689     * Snapshot names follow the same naming constraints as tables in HBase.
2690     * @param snapshotName name of the snapshot to be created
2691     * @param tableName name of the table for which snapshot is created
2692     * @throws IOException if a remote or network exception occurs
2693     * @throws SnapshotCreationException if snapshot creation failed
2694     * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2695     */
2696   public void snapshot(final byte[] snapshotName,
2697                        final TableName tableName) throws IOException,
2698       SnapshotCreationException, IllegalArgumentException {
2699     snapshot(Bytes.toString(snapshotName), tableName, SnapshotDescription.Type.FLUSH);
2700   }
2701 
2702   public void snapshot(final byte[] snapshotName,
2703                        final byte[] tableName) throws IOException,
2704       SnapshotCreationException, IllegalArgumentException {
2705     snapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName),
2706         SnapshotDescription.Type.FLUSH);
2707   }
2708 
2709   /**
2710    * Create typed snapshot of the table.
2711    * <p>
2712    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2713    * snapshot with the same name (even a different type or with different parameters) will fail with
2714    * a {@link SnapshotCreationException} indicating the duplicate naming.
2715    * <p>
2716    * Snapshot names follow the same naming constraints as tables in HBase. See
2717    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2718    * <p>
2719    * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other
2720    *          snapshots stored on the cluster
2721    * @param tableName name of the table to snapshot
2722    * @param type type of snapshot to take
2723    * @throws IOException we fail to reach the master
2724    * @throws SnapshotCreationException if snapshot creation failed
2725    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2726    */
2727   public void snapshot(final String snapshotName,
2728                        final TableName tableName,
2729                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2730       IllegalArgumentException {
2731     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
2732     builder.setTable(tableName.getNameAsString());
2733     builder.setName(snapshotName);
2734     builder.setType(type);
2735     snapshot(builder.build());
2736   }
2737 
2738   public void snapshot(final String snapshotName,
2739                        final String tableName,
2740                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2741       IllegalArgumentException {
2742     snapshot(snapshotName, TableName.valueOf(tableName), type);
2743   }
2744 
2745   public void snapshot(final String snapshotName,
2746                        final byte[] tableName,
2747                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2748       IllegalArgumentException {
2749     snapshot(snapshotName, TableName.valueOf(tableName), type);
2750   }
2751 
2752   /**
2753    * Take a snapshot and wait for the server to complete that snapshot (blocking).
2754    * <p>
2755    * Only a single snapshot should be taken at a time for an instance of HBase, or results may be
2756    * undefined (you can tell multiple HBase clusters to snapshot at the same time, but only one at a
2757    * time for a single cluster).
2758    * <p>
2759    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2760    * snapshot with the same name (even a different type or with different parameters) will fail with
2761    * a {@link SnapshotCreationException} indicating the duplicate naming.
2762    * <p>
2763    * Snapshot names follow the same naming constraints as tables in HBase. See
2764    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2765    * <p>
2766    * You should probably use {@link #snapshot(String, String)} or {@link #snapshot(byte[], byte[])}
2767    * unless you are sure about the type of snapshot that you want to take.
2768    * @param snapshot snapshot to take
2769    * @throws IOException or we lose contact with the master.
2770    * @throws SnapshotCreationException if snapshot failed to be taken
2771    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2772    */
2773   public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException,
2774       IllegalArgumentException {
2775     // actually take the snapshot
2776     SnapshotResponse response = takeSnapshotAsync(snapshot);
2777     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot)
2778         .build();
2779     IsSnapshotDoneResponse done = null;
2780     long start = EnvironmentEdgeManager.currentTimeMillis();
2781     long max = response.getExpectedTimeout();
2782     long maxPauseTime = max / this.numRetries;
2783     int tries = 0;
2784     LOG.debug("Waiting a max of " + max + " ms for snapshot '" +
2785         ClientSnapshotDescriptionUtils.toString(snapshot) + "'' to complete. (max " +
2786         maxPauseTime + " ms per retry)");
2787     while (tries == 0
2788         || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done.getDone())) {
2789       try {
2790         // sleep a backoff <= pauseTime amount
2791         long sleep = getPauseTime(tries++);
2792         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2793         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
2794           "ms while waiting for snapshot completion.");
2795         Thread.sleep(sleep);
2796 
2797       } catch (InterruptedException e) {
2798         LOG.debug("Interrupted while waiting for snapshot " + snapshot + " to complete");
2799         Thread.currentThread().interrupt();
2800       }
2801       LOG.debug("Getting current status of snapshot from master...");
2802       done = executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2803         @Override
2804         public IsSnapshotDoneResponse call() throws ServiceException {
2805           return master.isSnapshotDone(null, request);
2806         }
2807       });
2808     };
2809     if (!done.getDone()) {
2810       throw new SnapshotCreationException("Snapshot '" + snapshot.getName()
2811           + "' wasn't completed in expectedTime:" + max + " ms", snapshot);
2812     }
2813   }
2814 
2815   /**
2816    * Take a snapshot without waiting for the server to complete that snapshot (asynchronous)
2817    * <p>
2818    * Only a single snapshot should be taken at a time, or results may be undefined.
2819    * @param snapshot snapshot to take
2820    * @return response from the server indicating the max time to wait for the snapshot
2821    * @throws IOException if the snapshot did not succeed or we lose contact with the master.
2822    * @throws SnapshotCreationException if snapshot creation failed
2823    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2824    */
2825   public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException,
2826       SnapshotCreationException {
2827     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
2828     final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot)
2829         .build();
2830     // run the snapshot on the master
2831     return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) {
2832       @Override
2833       public SnapshotResponse call() throws ServiceException {
2834         return master.snapshot(null, request);
2835       }
2836     });
2837   }
2838 
2839   /**
2840    * Check the current state of the passed snapshot.
2841    * <p>
2842    * There are three possible states:
2843    * <ol>
2844    * <li>running - returns <tt>false</tt></li>
2845    * <li>finished - returns <tt>true</tt></li>
2846    * <li>finished with error - throws the exception that caused the snapshot to fail</li>
2847    * </ol>
2848    * <p>
2849    * The cluster only knows about the most recent snapshot. Therefore, if another snapshot has been
2850    * run/started since the snapshot your are checking, you will recieve an
2851    * {@link UnknownSnapshotException}.
2852    * @param snapshot description of the snapshot to check
2853    * @return <tt>true</tt> if the snapshot is completed, <tt>false</tt> if the snapshot is still
2854    *         running
2855    * @throws IOException if we have a network issue
2856    * @throws HBaseSnapshotException if the snapshot failed
2857    * @throws UnknownSnapshotException if the requested snapshot is unknown
2858    */
2859   public boolean isSnapshotFinished(final SnapshotDescription snapshot)
2860       throws IOException, HBaseSnapshotException, UnknownSnapshotException {
2861 
2862     return executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2863       @Override
2864       public IsSnapshotDoneResponse call() throws ServiceException {
2865         return master.isSnapshotDone(null,
2866           IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build());
2867       }
2868     }).getDone();
2869   }
2870 
2871   /**
2872    * Restore the specified snapshot on the original table. (The table must be disabled)
2873    * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
2874    * is set to true, a snapshot of the current table is taken
2875    * before executing the restore operation.
2876    * In case of restore failure, the failsafe snapshot will be restored.
2877    * If the restore completes without problem the failsafe snapshot is deleted.
2878    *
2879    * @param snapshotName name of the snapshot to restore
2880    * @throws IOException if a remote or network exception occurs
2881    * @throws RestoreSnapshotException if snapshot failed to be restored
2882    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2883    */
2884   public void restoreSnapshot(final byte[] snapshotName)
2885       throws IOException, RestoreSnapshotException {
2886     restoreSnapshot(Bytes.toString(snapshotName));
2887   }
2888 
2889   /**
2890    * Restore the specified snapshot on the original table. (The table must be disabled)
2891    * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
2892    * is set to true, a snapshot of the current table is taken
2893    * before executing the restore operation.
2894    * In case of restore failure, the failsafe snapshot will be restored.
2895    * If the restore completes without problem the failsafe snapshot is deleted.
2896    *
2897    * @param snapshotName name of the snapshot to restore
2898    * @throws IOException if a remote or network exception occurs
2899    * @throws RestoreSnapshotException if snapshot failed to be restored
2900    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2901    */
2902   public void restoreSnapshot(final String snapshotName)
2903       throws IOException, RestoreSnapshotException {
2904     boolean takeFailSafeSnapshot =
2905       conf.getBoolean("hbase.snapshot.restore.take.failsafe.snapshot", false);
2906     restoreSnapshot(snapshotName, takeFailSafeSnapshot);
2907   }
2908 
2909   /**
2910    * Restore the specified snapshot on the original table. (The table must be disabled)
2911    * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
2912    * before executing the restore operation.
2913    * In case of restore failure, the failsafe snapshot will be restored.
2914    * If the restore completes without problem the failsafe snapshot is deleted.
2915    *
2916    * The failsafe snapshot name is configurable by using the property
2917    * "hbase.snapshot.restore.failsafe.name".
2918    *
2919    * @param snapshotName name of the snapshot to restore
2920    * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
2921    * @throws IOException if a remote or network exception occurs
2922    * @throws RestoreSnapshotException if snapshot failed to be restored
2923    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2924    */
2925   public void restoreSnapshot(final byte[] snapshotName, final boolean takeFailSafeSnapshot)
2926       throws IOException, RestoreSnapshotException {
2927     restoreSnapshot(Bytes.toString(snapshotName), takeFailSafeSnapshot);
2928   }
2929 
2930   /**
2931    * Restore the specified snapshot on the original table. (The table must be disabled)
2932    * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
2933    * before executing the restore operation.
2934    * In case of restore failure, the failsafe snapshot will be restored.
2935    * If the restore completes without problem the failsafe snapshot is deleted.
2936    *
2937    * The failsafe snapshot name is configurable by using the property
2938    * "hbase.snapshot.restore.failsafe.name".
2939    *
2940    * @param snapshotName name of the snapshot to restore
2941    * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
2942    * @throws IOException if a remote or network exception occurs
2943    * @throws RestoreSnapshotException if snapshot failed to be restored
2944    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2945    */
2946   public void restoreSnapshot(final String snapshotName, boolean takeFailSafeSnapshot)
2947       throws IOException, RestoreSnapshotException {
2948     TableName tableName = null;
2949     for (SnapshotDescription snapshotInfo: listSnapshots()) {
2950       if (snapshotInfo.getName().equals(snapshotName)) {
2951         tableName = TableName.valueOf(snapshotInfo.getTable());
2952         break;
2953       }
2954     }
2955 
2956     if (tableName == null) {
2957       throw new RestoreSnapshotException(
2958         "Unable to find the table name for snapshot=" + snapshotName);
2959     }
2960 
2961     // The table does not exists, switch to clone.
2962     if (!tableExists(tableName)) {
2963       try {
2964         cloneSnapshot(snapshotName, tableName);
2965       } catch (InterruptedException e) {
2966         throw new InterruptedIOException("Interrupted when restoring a nonexistent table: " +
2967           e.getMessage());
2968       }
2969       return;
2970     }
2971 
2972     // Check if the table is disabled
2973     if (!isTableDisabled(tableName)) {
2974       throw new TableNotDisabledException(tableName);
2975     }
2976 
2977     // Take a snapshot of the current state
2978     String failSafeSnapshotSnapshotName = null;
2979     if (takeFailSafeSnapshot) {
2980       failSafeSnapshotSnapshotName = conf.get("hbase.snapshot.restore.failsafe.name",
2981         "hbase-failsafe-{snapshot.name}-{restore.timestamp}");
2982       failSafeSnapshotSnapshotName = failSafeSnapshotSnapshotName
2983         .replace("{snapshot.name}", snapshotName)
2984         .replace("{table.name}", tableName.toString().replace(TableName.NAMESPACE_DELIM, '.'))
2985         .replace("{restore.timestamp}", String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
2986       LOG.info("Taking restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
2987       snapshot(failSafeSnapshotSnapshotName, tableName);
2988     }
2989 
2990     try {
2991       // Restore snapshot
2992       internalRestoreSnapshot(snapshotName, tableName);
2993     } catch (IOException e) {
2994       // Somthing went wrong during the restore...
2995       // if the pre-restore snapshot is available try to rollback
2996       if (takeFailSafeSnapshot) {
2997         try {
2998           internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName);
2999           String msg = "Restore snapshot=" + snapshotName +
3000             " failed. Rollback to snapshot=" + failSafeSnapshotSnapshotName + " succeeded.";
3001           LOG.error(msg, e);
3002           throw new RestoreSnapshotException(msg, e);
3003         } catch (IOException ex) {
3004           String msg = "Failed to restore and rollback to snapshot=" + failSafeSnapshotSnapshotName;
3005           LOG.error(msg, ex);
3006           throw new RestoreSnapshotException(msg, e);
3007         }
3008       } else {
3009         throw new RestoreSnapshotException("Failed to restore snapshot=" + snapshotName, e);
3010       }
3011     }
3012 
3013     // If the restore is succeeded, delete the pre-restore snapshot
3014     if (takeFailSafeSnapshot) {
3015       try {
3016         LOG.info("Deleting restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
3017         deleteSnapshot(failSafeSnapshotSnapshotName);
3018       } catch (IOException e) {
3019         LOG.error("Unable to remove the failsafe snapshot: " + failSafeSnapshotSnapshotName, e);
3020       }
3021     }
3022   }
3023 
3024   /**
3025    * Create a new table by cloning the snapshot content.
3026    *
3027    * @param snapshotName name of the snapshot to be cloned
3028    * @param tableName name of the table where the snapshot will be restored
3029    * @throws IOException if a remote or network exception occurs
3030    * @throws TableExistsException if table to be created already exists
3031    * @throws RestoreSnapshotException if snapshot failed to be cloned
3032    * @throws IllegalArgumentException if the specified table has not a valid name
3033    */
3034   public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName)
3035       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3036     cloneSnapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName));
3037   }
3038 
3039   /**
3040    * Create a new table by cloning the snapshot content.
3041    *
3042    * @param snapshotName name of the snapshot to be cloned
3043    * @param tableName name of the table where the snapshot will be restored
3044    * @throws IOException if a remote or network exception occurs
3045    * @throws TableExistsException if table to be created already exists
3046    * @throws RestoreSnapshotException if snapshot failed to be cloned
3047    * @throws IllegalArgumentException if the specified table has not a valid name
3048    */
3049   public void cloneSnapshot(final byte[] snapshotName, final TableName tableName)
3050       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3051     cloneSnapshot(Bytes.toString(snapshotName), tableName);
3052   }
3053 
3054 
3055 
3056   /**
3057    * Create a new table by cloning the snapshot content.
3058    *
3059    * @param snapshotName name of the snapshot to be cloned
3060    * @param tableName name of the table where the snapshot will be restored
3061    * @throws IOException if a remote or network exception occurs
3062    * @throws TableExistsException if table to be created already exists
3063    * @throws RestoreSnapshotException if snapshot failed to be cloned
3064    * @throws IllegalArgumentException if the specified table has not a valid name
3065    */
3066   public void cloneSnapshot(final String snapshotName, final String tableName)
3067       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3068     cloneSnapshot(snapshotName, TableName.valueOf(tableName));
3069   }
3070 
3071   /**
3072    * Create a new table by cloning the snapshot content.
3073    *
3074    * @param snapshotName name of the snapshot to be cloned
3075    * @param tableName name of the table where the snapshot will be restored
3076    * @throws IOException if a remote or network exception occurs
3077    * @throws TableExistsException if table to be created already exists
3078    * @throws RestoreSnapshotException if snapshot failed to be cloned
3079    * @throws IllegalArgumentException if the specified table has not a valid name
3080    */
3081   public void cloneSnapshot(final String snapshotName, final TableName tableName)
3082       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3083     if (tableExists(tableName)) {
3084       throw new TableExistsException(tableName);
3085     }
3086     internalRestoreSnapshot(snapshotName, tableName);
3087     waitUntilTableIsEnabled(tableName);
3088   }
3089 
3090   /**
3091    * Execute a distributed procedure on a cluster.
3092    *
3093    * @param signature A distributed procedure is uniquely identified
3094    * by its signature (default the root ZK node name of the procedure).
3095    * @param instance The instance name of the procedure. For some procedures, this parameter is
3096    * optional.
3097    * @param props Property/Value pairs of properties passing to the procedure
3098    */
3099   public void execProcedure(String signature, String instance,
3100       Map<String, String> props) throws IOException {
3101     ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
3102     builder.setSignature(signature).setInstance(instance);
3103     for (String key : props.keySet()) {
3104       NameStringPair pair = NameStringPair.newBuilder().setName(key)
3105           .setValue(props.get(key)).build();
3106       builder.addConfiguration(pair);
3107     }
3108 
3109     final ExecProcedureRequest request = ExecProcedureRequest.newBuilder()
3110         .setProcedure(builder.build()).build();
3111     // run the procedure on the master
3112     ExecProcedureResponse response = executeCallable(new MasterCallable<ExecProcedureResponse>(
3113         getConnection()) {
3114       @Override
3115       public ExecProcedureResponse call() throws ServiceException {
3116         return master.execProcedure(null, request);
3117       }
3118     });
3119 
3120     long start = EnvironmentEdgeManager.currentTimeMillis();
3121     long max = response.getExpectedTimeout();
3122     long maxPauseTime = max / this.numRetries;
3123     int tries = 0;
3124     LOG.debug("Waiting a max of " + max + " ms for procedure '" +
3125         signature + " : " + instance + "'' to complete. (max " + maxPauseTime + " ms per retry)");
3126     boolean done = false;
3127     while (tries == 0
3128         || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done)) {
3129       try {
3130         // sleep a backoff <= pauseTime amount
3131         long sleep = getPauseTime(tries++);
3132         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
3133         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
3134           "ms while waiting for procedure completion.");
3135         Thread.sleep(sleep);
3136 
3137       } catch (InterruptedException e) {
3138         LOG.debug("Interrupted while waiting for procedure " + signature + " to complete");
3139         Thread.currentThread().interrupt();
3140       }
3141       LOG.debug("Getting current status of procedure from master...");
3142       done = isProcedureFinished(signature, instance, props);
3143     }
3144     if (!done) {
3145       throw new IOException("Procedure '" + signature + " : " + instance
3146           + "' wasn't completed in expectedTime:" + max + " ms");
3147     }
3148   }
3149 
3150   /**
3151    * Check the current state of the specified procedure.
3152    * <p>
3153    * There are three possible states:
3154    * <ol>
3155    * <li>running - returns <tt>false</tt></li>
3156    * <li>finished - returns <tt>true</tt></li>
3157    * <li>finished with error - throws the exception that caused the procedure to fail</li>
3158    * </ol>
3159    * <p>
3160    *
3161    * @param signature The signature that uniquely identifies a procedure
3162    * @param instance The instance name of the procedure
3163    * @param props Property/Value pairs of properties passing to the procedure
3164    * @return true if the specified procedure is finished successfully, false if it is still running
3165    * @throws IOException if the specified procedure finished with error
3166    */
3167   public boolean isProcedureFinished(String signature, String instance, Map<String, String> props)
3168       throws IOException {
3169     final ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
3170     builder.setSignature(signature).setInstance(instance);
3171     for (String key : props.keySet()) {
3172       NameStringPair pair = NameStringPair.newBuilder().setName(key)
3173           .setValue(props.get(key)).build();
3174       builder.addConfiguration(pair);
3175     }
3176     final ProcedureDescription desc = builder.build();
3177     return executeCallable(
3178         new MasterCallable<IsProcedureDoneResponse>(getConnection()) {
3179           @Override
3180           public IsProcedureDoneResponse call() throws ServiceException {
3181             return master.isProcedureDone(null, IsProcedureDoneRequest
3182                 .newBuilder().setProcedure(desc).build());
3183           }
3184         }).getDone();
3185   }
3186 
3187   /**
3188    * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
3189    * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
3190    * create an HTable instance to this table before it is available.
3191    * @param snapshotName snapshot to restore
3192    * @param tableName table name to restore the snapshot on
3193    * @throws IOException if a remote or network exception occurs
3194    * @throws RestoreSnapshotException if snapshot failed to be restored
3195    * @throws IllegalArgumentException if the restore request is formatted incorrectly
3196    */
3197   private void internalRestoreSnapshot(final String snapshotName, final TableName
3198       tableName)
3199       throws IOException, RestoreSnapshotException {
3200     SnapshotDescription snapshot = SnapshotDescription.newBuilder()
3201         .setName(snapshotName).setTable(tableName.getNameAsString()).build();
3202 
3203     // actually restore the snapshot
3204     internalRestoreSnapshotAsync(snapshot);
3205 
3206     final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
3207         .setSnapshot(snapshot).build();
3208     IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
3209         .setDone(false).buildPartial();
3210     final long maxPauseTime = 5000;
3211     int tries = 0;
3212     while (!done.getDone()) {
3213       try {
3214         // sleep a backoff <= pauseTime amount
3215         long sleep = getPauseTime(tries++);
3216         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
3217         LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
3218         Thread.sleep(sleep);
3219       } catch (InterruptedException e) {
3220         LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete");
3221         Thread.currentThread().interrupt();
3222       }
3223       LOG.debug("Getting current status of snapshot restore from master...");
3224       done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
3225           getConnection()) {
3226         @Override
3227         public IsRestoreSnapshotDoneResponse call() throws ServiceException {
3228           return master.isRestoreSnapshotDone(null, request);
3229         }
3230       });
3231     }
3232     if (!done.getDone()) {
3233       throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
3234     }
3235   }
3236 
3237   /**
3238    * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
3239    * <p>
3240    * Only a single snapshot should be restored at a time, or results may be undefined.
3241    * @param snapshot snapshot to restore
3242    * @return response from the server indicating the max time to wait for the snapshot
3243    * @throws IOException if a remote or network exception occurs
3244    * @throws RestoreSnapshotException if snapshot failed to be restored
3245    * @throws IllegalArgumentException if the restore request is formatted incorrectly
3246    */
3247   private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
3248       throws IOException, RestoreSnapshotException {
3249     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
3250 
3251     final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
3252         .build();
3253 
3254     // run the snapshot restore on the master
3255     return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
3256       @Override
3257       public RestoreSnapshotResponse call() throws ServiceException {
3258         return master.restoreSnapshot(null, request);
3259       }
3260     });
3261   }
3262 
3263   /**
3264    * List completed snapshots.
3265    * @return a list of snapshot descriptors for completed snapshots
3266    * @throws IOException if a network error occurs
3267    */
3268   public List<SnapshotDescription> listSnapshots() throws IOException {
3269     return executeCallable(new MasterCallable<List<SnapshotDescription>>(getConnection()) {
3270       @Override
3271       public List<SnapshotDescription> call() throws ServiceException {
3272         return master.getCompletedSnapshots(null, GetCompletedSnapshotsRequest.newBuilder().build())
3273             .getSnapshotsList();
3274       }
3275     });
3276   }
3277 
3278   /**
3279    * List all the completed snapshots matching the given regular expression.
3280    *
3281    * @param regex The regular expression to match against
3282    * @return - returns a List of SnapshotDescription
3283    * @throws IOException if a remote or network exception occurs
3284    */
3285   public List<SnapshotDescription> listSnapshots(String regex) throws IOException {
3286     return listSnapshots(Pattern.compile(regex));
3287   }
3288 
3289   /**
3290    * List all the completed snapshots matching the given pattern.
3291    *
3292    * @param pattern The compiled regular expression to match against
3293    * @return - returns a List of SnapshotDescription
3294    * @throws IOException if a remote or network exception occurs
3295    */
3296   public List<SnapshotDescription> listSnapshots(Pattern pattern) throws IOException {
3297     List<SnapshotDescription> matched = new LinkedList<SnapshotDescription>();
3298     List<SnapshotDescription> snapshots = listSnapshots();
3299     for (SnapshotDescription snapshot : snapshots) {
3300       if (pattern.matcher(snapshot.getName()).matches()) {
3301         matched.add(snapshot);
3302       }
3303     }
3304     return matched;
3305   }
3306 
3307   /**
3308    * Delete an existing snapshot.
3309    * @param snapshotName name of the snapshot
3310    * @throws IOException if a remote or network exception occurs
3311    */
3312   public void deleteSnapshot(final byte[] snapshotName) throws IOException {
3313     deleteSnapshot(Bytes.toString(snapshotName));
3314   }
3315 
3316   /**
3317    * Delete an existing snapshot.
3318    * @param snapshotName name of the snapshot
3319    * @throws IOException if a remote or network exception occurs
3320    */
3321   public void deleteSnapshot(final String snapshotName) throws IOException {
3322     // make sure the snapshot is possibly valid
3323     TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(snapshotName));
3324     // do the delete
3325     executeCallable(new MasterCallable<Void>(getConnection()) {
3326       @Override
3327       public Void call() throws ServiceException {
3328         master.deleteSnapshot(null,
3329           DeleteSnapshotRequest.newBuilder().
3330             setSnapshot(SnapshotDescription.newBuilder().setName(snapshotName).build()).build());
3331         return null;
3332       }
3333     });
3334   }
3335 
3336   /**
3337    * Delete existing snapshots whose names match the pattern passed.
3338    * @param regex The regular expression to match against
3339    * @throws IOException if a remote or network exception occurs
3340    */
3341   public void deleteSnapshots(final String regex) throws IOException {
3342     deleteSnapshots(Pattern.compile(regex));
3343   }
3344 
3345   /**
3346    * Delete existing snapshots whose names match the pattern passed.
3347    * @param pattern pattern for names of the snapshot to match
3348    * @throws IOException if a remote or network exception occurs
3349    */
3350   public void deleteSnapshots(final Pattern pattern) throws IOException {
3351     List<SnapshotDescription> snapshots = listSnapshots(pattern);
3352     for (final SnapshotDescription snapshot : snapshots) {
3353        try {
3354          internalDeleteSnapshot(snapshot);
3355        } catch (IOException ex) {
3356          LOG.info(
3357            "Failed to delete snapshot " + snapshot.getName() + " for table " + snapshot.getTable(),
3358            ex);
3359        }
3360     }
3361   }
3362 
3363   private void internalDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
3364     executeCallable(new MasterCallable<Void>(getConnection()) {
3365       @Override
3366       public Void call() throws ServiceException {
3367         this.master.deleteSnapshot(null, DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot)
3368             .build());
3369         return null;
3370       }
3371     });
3372   }
3373 
3374   /**
3375    * Parent of {@link MasterCallable} and {@link MasterCallable}.
3376    * Has common methods.
3377    * @param <V>
3378    */
3379   abstract static class MasterCallable<V> implements RetryingCallable<V>, Closeable {
3380     protected HConnection connection;
3381     protected MasterKeepAliveConnection master;
3382 
3383     public MasterCallable(final HConnection connection) {
3384       this.connection = connection;
3385     }
3386 
3387     @Override
3388     public void prepare(boolean reload) throws IOException {
3389       this.master = this.connection.getKeepAliveMasterService();
3390     }
3391 
3392     @Override
3393     public void close() throws IOException {
3394       // The above prepare could fail but this would still be called though masterAdmin is null
3395       if (this.master != null) this.master.close();
3396     }
3397 
3398     @Override
3399     public void throwable(Throwable t, boolean retrying) {
3400     }
3401 
3402     @Override
3403     public String getExceptionMessageAdditionalDetail() {
3404       return "";
3405     }
3406 
3407     @Override
3408     public long sleep(long pause, int tries) {
3409       return ConnectionUtils.getPauseTime(pause, tries);
3410     }
3411   }
3412 
3413   private <V> V executeCallable(MasterCallable<V> callable) throws IOException {
3414     RpcRetryingCaller<V> caller = rpcCallerFactory.newCaller();
3415     try {
3416       return caller.callWithRetries(callable);
3417     } finally {
3418       callable.close();
3419     }
3420   }
3421 
3422   /**
3423    * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
3424    * connected to the active master.
3425    *
3426    * <p>
3427    * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
3428    * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
3429    * </p>
3430    *
3431    * <div style="background-color: #cccccc; padding: 2px">
3432    * <blockquote><pre>
3433    * CoprocessorRpcChannel channel = myAdmin.coprocessorService();
3434    * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
3435    * MyCallRequest request = MyCallRequest.newBuilder()
3436    *     ...
3437    *     .build();
3438    * MyCallResponse response = service.myCall(null, request);
3439    * </pre></blockquote></div>
3440    *
3441    * @return A MasterCoprocessorRpcChannel instance
3442    */
3443   public CoprocessorRpcChannel coprocessorService() {
3444     return new MasterCoprocessorRpcChannel(connection);
3445   }
3446 
3447   /**
3448    * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
3449    * connected to the passed region server.
3450    *
3451    * <p>
3452    * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
3453    * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
3454    * </p>
3455    *
3456    * <div style="background-color: #cccccc; padding: 2px">
3457    * <blockquote><pre>
3458    * CoprocessorRpcChannel channel = myAdmin.coprocessorService(serverName);
3459    * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
3460    * MyCallRequest request = MyCallRequest.newBuilder()
3461    *     ...
3462    *     .build();
3463    * MyCallResponse response = service.myCall(null, request);
3464    * </pre></blockquote></div>
3465    *
3466    * @param sn the server name to which the endpoint call is made
3467    * @return A RegionServerCoprocessorRpcChannel instance
3468    */
3469   public CoprocessorRpcChannel coprocessorService(ServerName sn) {
3470     return new RegionServerCoprocessorRpcChannel(connection, sn);
3471   }
3472 
3473   /**
3474    * Truncate a table. Synchronous operation.
3475    * @param tableName name of table to truncate
3476    * @param preserveSplits True if the splits should be preserved
3477    * @throws IOException if a remote or network exception occurs
3478    */
3479   public void truncateTable(final TableName tableName, final boolean preserveSplits)
3480       throws IOException {
3481     executeCallable(new MasterCallable<Void>(getConnection()) {
3482       @Override
3483       public Void call() throws ServiceException {
3484         LOG.info("Started truncate of " + tableName);
3485         TruncateTableRequest req = RequestConverter.buildTruncateTableRequest(
3486         tableName, preserveSplits);
3487         master.truncateTable(null, req);
3488         return null;
3489       }
3490     });
3491   }
3492 
3493 }