View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  import java.net.SocketTimeoutException;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.atomic.AtomicInteger;
32  import java.util.concurrent.atomic.AtomicReference;
33  import java.util.regex.Pattern;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.Abortable;
39  import org.apache.hadoop.hbase.ClusterStatus;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HBaseIOException;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HRegionLocation;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.MasterNotRunningException;
48  import org.apache.hadoop.hbase.NamespaceDescriptor;
49  import org.apache.hadoop.hbase.NotServingRegionException;
50  import org.apache.hadoop.hbase.RegionException;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.TableExistsException;
53  import org.apache.hadoop.hbase.TableName;
54  import org.apache.hadoop.hbase.TableNotDisabledException;
55  import org.apache.hadoop.hbase.TableNotEnabledException;
56  import org.apache.hadoop.hbase.TableNotFoundException;
57  import org.apache.hadoop.hbase.UnknownRegionException;
58  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
59  import org.apache.hadoop.hbase.catalog.CatalogTracker;
60  import org.apache.hadoop.hbase.catalog.MetaReader;
61  import org.apache.hadoop.hbase.classification.InterfaceAudience;
62  import org.apache.hadoop.hbase.classification.InterfaceStability;
63  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
64  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
65  import org.apache.hadoop.hbase.exceptions.DeserializationException;
66  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
67  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
68  import org.apache.hadoop.hbase.ipc.MasterCoprocessorRpcChannel;
69  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
70  import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel;
71  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
72  import org.apache.hadoop.hbase.protobuf.RequestConverter;
73  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
86  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
87  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
88  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
89  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
90  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
91  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
92  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
93  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
94  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnRequest;
98  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotRequest;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRequest;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureResponse;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
117 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
118 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
119 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
120 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest;
121 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest;
122 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
124 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRequest;
125 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
126 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
127 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
128 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
129 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
130 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest;
131 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
132 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
133 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableRequest;
134 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
135 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
136 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
137 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
138 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
139 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
140 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
141 import org.apache.hadoop.hbase.util.Addressing;
142 import org.apache.hadoop.hbase.util.Bytes;
143 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
144 import org.apache.hadoop.hbase.util.Pair;
145 import org.apache.hadoop.ipc.RemoteException;
146 import org.apache.hadoop.util.StringUtils;
147 import org.apache.zookeeper.KeeperException;
148 
149 import com.google.common.annotations.VisibleForTesting;
150 import com.google.protobuf.ByteString;
151 import com.google.protobuf.ServiceException;
152 
153 /**
154  * Provides an interface to manage HBase database table metadata + general
155  * administrative functions.  Use HBaseAdmin to create, drop, list, enable and
156  * disable tables. Use it also to add and drop table column families.
157  *
158  * <p>See {@link HTable} to add, update, and delete data from an individual table.
159  * <p>Currently HBaseAdmin instances are not expected to be long-lived.  For
160  * example, an HBaseAdmin instance will not ride over a Master restart.
161  */
162 @InterfaceAudience.Public
163 @InterfaceStability.Evolving
164 public class HBaseAdmin implements Abortable, Closeable {
165   private static final Log LOG = LogFactory.getLog(HBaseAdmin.class);
166 
167   // We use the implementation class rather then the interface because we
168   //  need the package protected functions to get the connection to master
169   private HConnection connection;
170 
171   private volatile Configuration conf;
172   private final long pause;
173   private final int numRetries;
174   // Some operations can take a long time such as disable of big table.
175   // numRetries is for 'normal' stuff... Multiply by this factor when
176   // want to wait a long time.
177   private final int retryLongerMultiplier;
178   private boolean aborted;
179   private boolean cleanupConnectionOnClose = false; // close the connection in close()
180   private boolean closed = false;
181 
182   private RpcRetryingCallerFactory rpcCallerFactory;
183 
184   /**
185    * Constructor.
186    * See {@link #HBaseAdmin(HConnection connection)}
187    *
188    * @param c Configuration object. Copied internally.
189    */
190   public HBaseAdmin(Configuration c)
191   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
192     // Will not leak connections, as the new implementation of the constructor
193     // does not throw exceptions anymore.
194     this(HConnectionManager.getConnection(new Configuration(c)));
195     this.cleanupConnectionOnClose = true;
196   }
197 
198  /**
199   * Constructor for externally managed HConnections.
200   * The connection to master will be created when required by admin functions.
201   *
202   * @param connection The HConnection instance to use
203   * @throws MasterNotRunningException, ZooKeeperConnectionException are not
204   *  thrown anymore but kept into the interface for backward api compatibility
205   */
206   public HBaseAdmin(HConnection connection)
207       throws MasterNotRunningException, ZooKeeperConnectionException {
208     this.conf = connection.getConfiguration();
209     this.connection = connection;
210 
211     this.pause = this.conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
212         HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
213     this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
214         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
215     this.retryLongerMultiplier = this.conf.getInt(
216         "hbase.client.retries.longer.multiplier", 10);
217     this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf);
218   }
219 
220   /**
221    * @return A new CatalogTracker instance; call {@link #cleanupCatalogTracker(CatalogTracker)}
222    * to cleanup the returned catalog tracker.
223    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
224    * @throws IOException
225    * @see #cleanupCatalogTracker(CatalogTracker)
226    */
227   @VisibleForTesting
228   synchronized CatalogTracker getCatalogTracker()
229   throws ZooKeeperConnectionException, IOException {
230     boolean succeeded = false;
231     CatalogTracker ct = null;
232     try {
233       ct = new CatalogTracker(this.conf);
234       startCatalogTracker(ct);
235       succeeded = true;
236     } catch (InterruptedException e) {
237       // Let it out as an IOE for now until we redo all so tolerate IEs
238       throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
239     } finally {
240       // If we did not succeed but created a catalogtracker, clean it up. CT has a ZK instance
241       // in it and we'll leak if we don't do the 'stop'.
242       if (!succeeded && ct != null) {
243         try {
244           ct.stop();
245         } catch (RuntimeException re) {
246           LOG.error("Failed to clean up HBase's internal catalog tracker after a failed initialization. " +
247             "We may have leaked network connections to ZooKeeper; they won't be cleaned up until " +
248             "the JVM exits. If you see a large number of stale connections to ZooKeeper this is likely " +
249             "the cause. The following exception details will be needed for assistance from the " +
250             "HBase community.", re);
251         }
252         ct = null;
253       }
254     }
255     return ct;
256   }
257 
258   @VisibleForTesting
259   CatalogTracker startCatalogTracker(final CatalogTracker ct)
260   throws IOException, InterruptedException {
261     ct.start();
262     return ct;
263   }
264 
265   @VisibleForTesting
266   void cleanupCatalogTracker(final CatalogTracker ct) {
267     ct.stop();
268   }
269 
270   @Override
271   public void abort(String why, Throwable e) {
272     // Currently does nothing but throw the passed message and exception
273     this.aborted = true;
274     throw new RuntimeException(why, e);
275   }
276 
277   @Override
278   public boolean isAborted(){
279     return this.aborted;
280   }
281 
282   /** @return HConnection used by this object. */
283   public HConnection getConnection() {
284     return connection;
285   }
286 
287   /** @return - true if the master server is running. Throws an exception
288    *  otherwise.
289    * @throws ZooKeeperConnectionException
290    * @throws MasterNotRunningException
291    */
292   public boolean isMasterRunning()
293   throws MasterNotRunningException, ZooKeeperConnectionException {
294     return connection.isMasterRunning();
295   }
296 
297   /**
298    * @param tableName Table to check.
299    * @return True if table exists already.
300    * @throws IOException
301    */
302   public boolean tableExists(final TableName tableName)
303   throws IOException {
304     boolean b = false;
305     CatalogTracker ct = getCatalogTracker();
306     try {
307       b = MetaReader.tableExists(ct, tableName);
308     } finally {
309       cleanupCatalogTracker(ct);
310     }
311     return b;
312   }
313 
314   public boolean tableExists(final byte[] tableName)
315   throws IOException {
316     return tableExists(TableName.valueOf(tableName));
317   }
318 
319   public boolean tableExists(final String tableName)
320   throws IOException {
321     return tableExists(TableName.valueOf(tableName));
322   }
323 
324   /**
325    * List all the userspace tables.  In other words, scan the hbase:meta table.
326    *
327    * If we wanted this to be really fast, we could implement a special
328    * catalog table that just contains table names and their descriptors.
329    * Right now, it only exists as part of the hbase:meta table's region info.
330    *
331    * @return - returns an array of HTableDescriptors
332    * @throws IOException if a remote or network exception occurs
333    */
334   public HTableDescriptor[] listTables() throws IOException {
335     return this.connection.listTables();
336   }
337 
338   /**
339    * List all the userspace tables matching the given pattern.
340    *
341    * @param pattern The compiled regular expression to match against
342    * @return - returns an array of HTableDescriptors
343    * @throws IOException if a remote or network exception occurs
344    * @see #listTables()
345    */
346   public HTableDescriptor[] listTables(Pattern pattern) throws IOException {
347     List<HTableDescriptor> matched = new LinkedList<HTableDescriptor>();
348     HTableDescriptor[] tables = listTables();
349     for (HTableDescriptor table : tables) {
350       if (pattern.matcher(table.getTableName().getNameAsString()).matches()) {
351         matched.add(table);
352       }
353     }
354     return matched.toArray(new HTableDescriptor[matched.size()]);
355   }
356 
357   /**
358    * List all the userspace tables matching the given regular expression.
359    *
360    * @param regex The regular expression to match against
361    * @return - returns an array of HTableDescriptors
362    * @throws IOException if a remote or network exception occurs
363    * @see #listTables(java.util.regex.Pattern)
364    */
365   public HTableDescriptor[] listTables(String regex) throws IOException {
366     return listTables(Pattern.compile(regex));
367   }
368 
369   /**
370    * List all of the names of userspace tables.
371    * @return String[] table names
372    * @throws IOException if a remote or network exception occurs
373    */
374   @Deprecated
375   public String[] getTableNames() throws IOException {
376     return this.connection.getTableNames();
377   }
378 
379   /**
380    * List all of the names of userspace tables matching the given regular expression.
381    * @param pattern The regular expression to match against
382    * @return String[] table names
383    * @throws IOException if a remote or network exception occurs
384    */
385   @Deprecated
386   public String[] getTableNames(Pattern pattern) throws IOException {
387     List<String> matched = new ArrayList<String>();
388     for (String name: this.connection.getTableNames()) {
389       if (pattern.matcher(name).matches()) {
390         matched.add(name);
391       }
392     }
393     return matched.toArray(new String[matched.size()]);
394   }
395 
396   /**
397    * List all of the names of userspace tables matching the given regular expression.
398    * @param regex The regular expression to match against
399    * @return String[] table names
400    * @throws IOException if a remote or network exception occurs
401    */
402   @Deprecated
403   public String[] getTableNames(String regex) throws IOException {
404     return getTableNames(Pattern.compile(regex));
405   }
406 
407   /**
408    * List all of the names of userspace tables.
409    * @return TableName[] table names
410    * @throws IOException if a remote or network exception occurs
411    */
412   public TableName[] listTableNames() throws IOException {
413     return this.connection.listTableNames();
414   }
415 
416   /**
417    * Method for getting the tableDescriptor
418    * @param tableName as a byte []
419    * @return the tableDescriptor
420    * @throws TableNotFoundException
421    * @throws IOException if a remote or network exception occurs
422    */
423   public HTableDescriptor getTableDescriptor(final TableName tableName)
424   throws TableNotFoundException, IOException {
425     return this.connection.getHTableDescriptor(tableName);
426   }
427 
428   public HTableDescriptor getTableDescriptor(final byte[] tableName)
429   throws TableNotFoundException, IOException {
430     return getTableDescriptor(TableName.valueOf(tableName));
431   }
432 
433   private long getPauseTime(int tries) {
434     int triesCount = tries;
435     if (triesCount >= HConstants.RETRY_BACKOFF.length) {
436       triesCount = HConstants.RETRY_BACKOFF.length - 1;
437     }
438     return this.pause * HConstants.RETRY_BACKOFF[triesCount];
439   }
440 
441   /**
442    * Creates a new table.
443    * Synchronous operation.
444    *
445    * @param desc table descriptor for table
446    *
447    * @throws IllegalArgumentException if the table name is reserved
448    * @throws MasterNotRunningException if master is not running
449    * @throws TableExistsException if table already exists (If concurrent
450    * threads, the table may have been created between test-for-existence
451    * and attempt-at-creation).
452    * @throws IOException if a remote or network exception occurs
453    */
454   public void createTable(HTableDescriptor desc)
455   throws IOException {
456     createTable(desc, null);
457   }
458 
459   /**
460    * Creates a new table with the specified number of regions.  The start key
461    * specified will become the end key of the first region of the table, and
462    * the end key specified will become the start key of the last region of the
463    * table (the first region has a null start key and the last region has a
464    * null end key).
465    *
466    * BigInteger math will be used to divide the key range specified into
467    * enough segments to make the required number of total regions.
468    *
469    * Synchronous operation.
470    *
471    * @param desc table descriptor for table
472    * @param startKey beginning of key range
473    * @param endKey end of key range
474    * @param numRegions the total number of regions to create
475    *
476    * @throws IllegalArgumentException if the table name is reserved
477    * @throws MasterNotRunningException if master is not running
478    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
479    * threads, the table may have been created between test-for-existence
480    * and attempt-at-creation).
481    * @throws IOException
482    */
483   public void createTable(HTableDescriptor desc, byte [] startKey,
484       byte [] endKey, int numRegions)
485   throws IOException {
486     if(numRegions < 3) {
487       throw new IllegalArgumentException("Must create at least three regions");
488     } else if(Bytes.compareTo(startKey, endKey) >= 0) {
489       throw new IllegalArgumentException("Start key must be smaller than end key");
490     }
491     if (numRegions == 3) {
492       createTable(desc, new byte[][]{startKey, endKey});
493       return;
494     }
495     byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
496     if(splitKeys == null || splitKeys.length != numRegions - 1) {
497       throw new IllegalArgumentException("Unable to split key range into enough regions");
498     }
499     createTable(desc, splitKeys);
500   }
501 
502   /**
503    * Creates a new table with an initial set of empty regions defined by the
504    * specified split keys.  The total number of regions created will be the
505    * number of split keys plus one. Synchronous operation.
506    * Note : Avoid passing empty split key.
507    *
508    * @param desc table descriptor for table
509    * @param splitKeys array of split keys for the initial regions of the table
510    *
511    * @throws IllegalArgumentException if the table name is reserved, if the split keys
512    * are repeated and if the split key has empty byte array.
513    * @throws MasterNotRunningException if master is not running
514    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
515    * threads, the table may have been created between test-for-existence
516    * and attempt-at-creation).
517    * @throws IOException
518    */
519   public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
520   throws IOException {
521     try {
522       createTableAsync(desc, splitKeys);
523     } catch (SocketTimeoutException ste) {
524       LOG.warn("Creating " + desc.getTableName() + " took too long", ste);
525     }
526     int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
527     int prevRegCount = 0;
528     boolean doneWithMetaScan = false;
529     for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier;
530       ++tries) {
531       if (!doneWithMetaScan) {
532         // Wait for new table to come on-line
533         final AtomicInteger actualRegCount = new AtomicInteger(0);
534         MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
535           @Override
536           public boolean processRow(Result rowResult) throws IOException {
537             HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
538             if (info == null) {
539               LOG.warn("No serialized HRegionInfo in " + rowResult);
540               return true;
541             }
542             if (!info.getTable().equals(desc.getTableName())) {
543               return false;
544             }
545             ServerName serverName = HRegionInfo.getServerName(rowResult);
546             // Make sure that regions are assigned to server
547             if (!(info.isOffline() || info.isSplit()) && serverName != null
548                 && serverName.getHostAndPort() != null) {
549               actualRegCount.incrementAndGet();
550             }
551             return true;
552           }
553         };
554         MetaScanner.metaScan(conf, connection, visitor, desc.getTableName());
555         if (actualRegCount.get() < numRegs) {
556           if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
557             throw new RegionOfflineException("Only " + actualRegCount.get() +
558               " of " + numRegs + " regions are online; retries exhausted.");
559           }
560           try { // Sleep
561             Thread.sleep(getPauseTime(tries));
562           } catch (InterruptedException e) {
563             throw new InterruptedIOException("Interrupted when opening" +
564               " regions; " + actualRegCount.get() + " of " + numRegs +
565               " regions processed so far");
566           }
567           if (actualRegCount.get() > prevRegCount) { // Making progress
568             prevRegCount = actualRegCount.get();
569             tries = -1;
570           }
571         } else {
572           doneWithMetaScan = true;
573           tries = -1;
574         }
575       } else if (isTableEnabled(desc.getTableName())) {
576         return;
577       } else {
578         try { // Sleep
579           Thread.sleep(getPauseTime(tries));
580         } catch (InterruptedException e) {
581           throw new InterruptedIOException("Interrupted when waiting" +
582             " for table to be enabled; meta scan was done");
583         }
584       }
585     }
586     throw new TableNotEnabledException(
587       "Retries exhausted while still waiting for table: "
588       + desc.getTableName() + " to be enabled");
589   }
590 
591   /**
592    * Creates a new table but does not block and wait for it to come online.
593    * Asynchronous operation.  To check if the table exists, use
594    * {@link #isTableAvailable} -- it is not safe to create an HTable
595    * instance to this table before it is available.
596    * Note : Avoid passing empty split key.
597    * @param desc table descriptor for table
598    *
599    * @throws IllegalArgumentException Bad table name, if the split keys
600    * are repeated and if the split key has empty byte array.
601    * @throws MasterNotRunningException if master is not running
602    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
603    * threads, the table may have been created between test-for-existence
604    * and attempt-at-creation).
605    * @throws IOException
606    */
607   public void createTableAsync(
608     final HTableDescriptor desc, final byte [][] splitKeys)
609   throws IOException {
610     if(desc.getTableName() == null) {
611       throw new IllegalArgumentException("TableName cannot be null");
612     }
613     if(splitKeys != null && splitKeys.length > 0) {
614       Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
615       // Verify there are no duplicate split keys
616       byte [] lastKey = null;
617       for(byte [] splitKey : splitKeys) {
618         if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) {
619           throw new IllegalArgumentException(
620               "Empty split key must not be passed in the split keys.");
621         }
622         if(lastKey != null && Bytes.equals(splitKey, lastKey)) {
623           throw new IllegalArgumentException("All split keys must be unique, " +
624             "found duplicate: " + Bytes.toStringBinary(splitKey) +
625             ", " + Bytes.toStringBinary(lastKey));
626         }
627         lastKey = splitKey;
628       }
629     }
630 
631     executeCallable(new MasterCallable<Void>(getConnection()) {
632       @Override
633       public Void call() throws ServiceException {
634         CreateTableRequest request = RequestConverter.buildCreateTableRequest(desc, splitKeys);
635         master.createTable(null, request);
636         return null;
637       }
638     });
639   }
640 
641   public void deleteTable(final String tableName) throws IOException {
642     deleteTable(TableName.valueOf(tableName));
643   }
644 
645   public void deleteTable(final byte[] tableName) throws IOException {
646     deleteTable(TableName.valueOf(tableName));
647   }
648 
649   /**
650    * Deletes a table.
651    * Synchronous operation.
652    *
653    * @param tableName name of table to delete
654    * @throws IOException if a remote or network exception occurs
655    */
656   public void deleteTable(final TableName tableName) throws IOException {
657     boolean tableExists = true;
658 
659     executeCallable(new MasterCallable<Void>(getConnection()) {
660       @Override
661       public Void call() throws ServiceException {
662         DeleteTableRequest req = RequestConverter.buildDeleteTableRequest(tableName);
663         master.deleteTable(null,req);
664         return null;
665       }
666     });
667 
668     int failures = 0;
669     // Wait until all regions deleted
670     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
671       try {
672         HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName);
673         Scan scan = MetaReader.getScanForTableName(tableName);
674         scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
675         ScanRequest request = RequestConverter.buildScanRequest(
676           firstMetaServer.getRegionInfo().getRegionName(), scan, 1, true);
677         Result[] values = null;
678         // Get a batch at a time.
679         ClientService.BlockingInterface server = connection.getClient(firstMetaServer
680             .getServerName());
681         PayloadCarryingRpcController controller = new PayloadCarryingRpcController();
682         try {
683           controller.setPriority(tableName);
684           ScanResponse response = server.scan(controller, request);
685           values = ResponseConverter.getResults(controller.cellScanner(), response);
686         } catch (ServiceException se) {
687           throw ProtobufUtil.getRemoteException(se);
688         }
689 
690         // let us wait until hbase:meta table is updated and
691         // HMaster removes the table from its HTableDescriptors
692         if (values == null || values.length == 0) {
693           tableExists = false;
694           GetTableDescriptorsResponse htds;
695           MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
696           try {
697             GetTableDescriptorsRequest req =
698               RequestConverter.buildGetTableDescriptorsRequest(tableName);
699             htds = master.getTableDescriptors(null, req);
700           } catch (ServiceException se) {
701             throw ProtobufUtil.getRemoteException(se);
702           } finally {
703             master.close();
704           }
705           tableExists = !htds.getTableSchemaList().isEmpty();
706           if (!tableExists) {
707             break;
708           }
709         }
710       } catch (IOException ex) {
711         failures++;
712         if(failures == numRetries - 1) {           // no more tries left
713           if (ex instanceof RemoteException) {
714             throw ((RemoteException) ex).unwrapRemoteException();
715           } else {
716             throw ex;
717           }
718         }
719       }
720       try {
721         Thread.sleep(getPauseTime(tries));
722       } catch (InterruptedException e) {
723         // continue
724       }
725     }
726 
727     if (tableExists) {
728       throw new IOException("Retries exhausted, it took too long to wait"+
729         " for the table " + tableName + " to be deleted.");
730     }
731     // Delete cached information to prevent clients from using old locations
732     this.connection.clearRegionCache(tableName);
733     LOG.info("Deleted " + tableName);
734   }
735 
736   /**
737    * Deletes tables matching the passed in pattern and wait on completion.
738    *
739    * Warning: Use this method carefully, there is no prompting and the effect is
740    * immediate. Consider using {@link #listTables(java.lang.String)} and
741    * {@link #deleteTable(byte[])}
742    *
743    * @param regex The regular expression to match table names against
744    * @return Table descriptors for tables that couldn't be deleted
745    * @throws IOException
746    * @see #deleteTables(java.util.regex.Pattern)
747    * @see #deleteTable(java.lang.String)
748    */
749   public HTableDescriptor[] deleteTables(String regex) throws IOException {
750     return deleteTables(Pattern.compile(regex));
751   }
752 
753   /**
754    * Delete tables matching the passed in pattern and wait on completion.
755    *
756    * Warning: Use this method carefully, there is no prompting and the effect is
757    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
758    * {@link #deleteTable(byte[])}
759    *
760    * @param pattern The pattern to match table names against
761    * @return Table descriptors for tables that couldn't be deleted
762    * @throws IOException
763    */
764   public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException {
765     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
766     for (HTableDescriptor table : listTables(pattern)) {
767       try {
768         deleteTable(table.getTableName());
769       } catch (IOException ex) {
770         LOG.info("Failed to delete table " + table.getTableName(), ex);
771         failed.add(table);
772       }
773     }
774     return failed.toArray(new HTableDescriptor[failed.size()]);
775   }
776 
777 
778   /**
779    * Enable a table.  May timeout.  Use {@link #enableTableAsync(byte[])}
780    * and {@link #isTableEnabled(byte[])} instead.
781    * The table has to be in disabled state for it to be enabled.
782    * @param tableName name of the table
783    * @throws IOException if a remote or network exception occurs
784    * There could be couple types of IOException
785    * TableNotFoundException means the table doesn't exist.
786    * TableNotDisabledException means the table isn't in disabled state.
787    * @see #isTableEnabled(byte[])
788    * @see #disableTable(byte[])
789    * @see #enableTableAsync(byte[])
790    */
791   public void enableTable(final TableName tableName)
792   throws IOException {
793     enableTableAsync(tableName);
794 
795     // Wait until all regions are enabled
796     waitUntilTableIsEnabled(tableName);
797 
798     LOG.info("Enabled table " + tableName);
799   }
800 
801   public void enableTable(final byte[] tableName)
802   throws IOException {
803     enableTable(TableName.valueOf(tableName));
804   }
805 
806   public void enableTable(final String tableName)
807   throws IOException {
808     enableTable(TableName.valueOf(tableName));
809   }
810 
811   /**
812    * Wait for the table to be enabled and available
813    * If enabling the table exceeds the retry period, an exception is thrown.
814    * @param tableName name of the table
815    * @throws IOException if a remote or network exception occurs or
816    *    table is not enabled after the retries period.
817    */
818   private void waitUntilTableIsEnabled(final TableName tableName) throws IOException {
819     boolean enabled = false;
820     long start = EnvironmentEdgeManager.currentTimeMillis();
821     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
822       try {
823         enabled = isTableEnabled(tableName);
824       } catch (TableNotFoundException tnfe) {
825         // wait for table to be created
826         enabled = false;
827       }
828       enabled = enabled && isTableAvailable(tableName);
829       if (enabled) {
830         break;
831       }
832       long sleep = getPauseTime(tries);
833       if (LOG.isDebugEnabled()) {
834         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
835           "enabled in " + tableName);
836       }
837       try {
838         Thread.sleep(sleep);
839       } catch (InterruptedException e) {
840         // Do this conversion rather than let it out because do not want to
841         // change the method signature.
842         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
843       }
844     }
845     if (!enabled) {
846       long msec = EnvironmentEdgeManager.currentTimeMillis() - start;
847       throw new IOException("Table '" + tableName +
848         "' not yet enabled, after " + msec + "ms.");
849     }
850   }
851 
852   /**
853    * Brings a table on-line (enables it).  Method returns immediately though
854    * enable of table may take some time to complete, especially if the table
855    * is large (All regions are opened as part of enabling process).  Check
856    * {@link #isTableEnabled(byte[])} to learn when table is fully online.  If
857    * table is taking too long to online, check server logs.
858    * @param tableName
859    * @throws IOException
860    * @since 0.90.0
861    */
862   public void enableTableAsync(final TableName tableName)
863   throws IOException {
864     TableName.isLegalFullyQualifiedTableName(tableName.getName());
865     executeCallable(new MasterCallable<Void>(getConnection()) {
866       @Override
867       public Void call() throws ServiceException {
868         LOG.info("Started enable of " + tableName);
869         EnableTableRequest req = RequestConverter.buildEnableTableRequest(tableName);
870         master.enableTable(null,req);
871         return null;
872       }
873     });
874   }
875 
876   public void enableTableAsync(final byte[] tableName)
877   throws IOException {
878     enableTable(TableName.valueOf(tableName));
879   }
880 
881   public void enableTableAsync(final String tableName)
882   throws IOException {
883     enableTableAsync(TableName.valueOf(tableName));
884   }
885 
886   /**
887    * Enable tables matching the passed in pattern and wait on completion.
888    *
889    * Warning: Use this method carefully, there is no prompting and the effect is
890    * immediate. Consider using {@link #listTables(java.lang.String)} and
891    * {@link #enableTable(byte[])}
892    *
893    * @param regex The regular expression to match table names against
894    * @throws IOException
895    * @see #enableTables(java.util.regex.Pattern)
896    * @see #enableTable(java.lang.String)
897    */
898   public HTableDescriptor[] enableTables(String regex) throws IOException {
899     return enableTables(Pattern.compile(regex));
900   }
901 
902   /**
903    * Enable tables matching the passed in pattern and wait on completion.
904    *
905    * Warning: Use this method carefully, there is no prompting and the effect is
906    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
907    * {@link #enableTable(byte[])}
908    *
909    * @param pattern The pattern to match table names against
910    * @throws IOException
911    */
912   public HTableDescriptor[] enableTables(Pattern pattern) throws IOException {
913     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
914     for (HTableDescriptor table : listTables(pattern)) {
915       if (isTableDisabled(table.getTableName())) {
916         try {
917           enableTable(table.getTableName());
918         } catch (IOException ex) {
919           LOG.info("Failed to enable table " + table.getTableName(), ex);
920           failed.add(table);
921         }
922       }
923     }
924     return failed.toArray(new HTableDescriptor[failed.size()]);
925   }
926 
927   /**
928    * Starts the disable of a table.  If it is being served, the master
929    * will tell the servers to stop serving it.  This method returns immediately.
930    * The disable of a table can take some time if the table is large (all
931    * regions are closed as part of table disable operation).
932    * Call {@link #isTableDisabled(byte[])} to check for when disable completes.
933    * If table is taking too long to online, check server logs.
934    * @param tableName name of table
935    * @throws IOException if a remote or network exception occurs
936    * @see #isTableDisabled(byte[])
937    * @see #isTableEnabled(byte[])
938    * @since 0.90.0
939    */
940   public void disableTableAsync(final TableName tableName) throws IOException {
941     TableName.isLegalFullyQualifiedTableName(tableName.getName());
942     executeCallable(new MasterCallable<Void>(getConnection()) {
943       @Override
944       public Void call() throws ServiceException {
945         LOG.info("Started disable of " + tableName);
946         DisableTableRequest req = RequestConverter.buildDisableTableRequest(tableName);
947         master.disableTable(null,req);
948         return null;
949       }
950     });
951   }
952 
953   public void disableTableAsync(final byte[] tableName) throws IOException {
954     disableTableAsync(TableName.valueOf(tableName));
955   }
956 
957   public void disableTableAsync(final String tableName) throws IOException {
958     disableTableAsync(TableName.valueOf(tableName));
959   }
960 
961   /**
962    * Disable table and wait on completion.  May timeout eventually.  Use
963    * {@link #disableTableAsync(byte[])} and {@link #isTableDisabled(String)}
964    * instead.
965    * The table has to be in enabled state for it to be disabled.
966    * @param tableName
967    * @throws IOException
968    * There could be couple types of IOException
969    * TableNotFoundException means the table doesn't exist.
970    * TableNotEnabledException means the table isn't in enabled state.
971    */
972   public void disableTable(final TableName tableName)
973   throws IOException {
974     disableTableAsync(tableName);
975     // Wait until table is disabled
976     boolean disabled = false;
977     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
978       disabled = isTableDisabled(tableName);
979       if (disabled) {
980         break;
981       }
982       long sleep = getPauseTime(tries);
983       if (LOG.isDebugEnabled()) {
984         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
985           "disabled in " + tableName);
986       }
987       try {
988         Thread.sleep(sleep);
989       } catch (InterruptedException e) {
990         // Do this conversion rather than let it out because do not want to
991         // change the method signature.
992         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
993       }
994     }
995     if (!disabled) {
996       throw new RegionException("Retries exhausted, it took too long to wait"+
997         " for the table " + tableName + " to be disabled.");
998     }
999     LOG.info("Disabled " + tableName);
1000   }
1001 
1002   public void disableTable(final byte[] tableName)
1003   throws IOException {
1004     disableTable(TableName.valueOf(tableName));
1005   }
1006 
1007   public void disableTable(final String tableName)
1008   throws IOException {
1009     disableTable(TableName.valueOf(tableName));
1010   }
1011 
1012   /**
1013    * Disable tables matching the passed in pattern and wait on completion.
1014    *
1015    * Warning: Use this method carefully, there is no prompting and the effect is
1016    * immediate. Consider using {@link #listTables(java.lang.String)} and
1017    * {@link #disableTable(byte[])}
1018    *
1019    * @param regex The regular expression to match table names against
1020    * @return Table descriptors for tables that couldn't be disabled
1021    * @throws IOException
1022    * @see #disableTables(java.util.regex.Pattern)
1023    * @see #disableTable(java.lang.String)
1024    */
1025   public HTableDescriptor[] disableTables(String regex) throws IOException {
1026     return disableTables(Pattern.compile(regex));
1027   }
1028 
1029   /**
1030    * Disable tables matching the passed in pattern and wait on completion.
1031    *
1032    * Warning: Use this method carefully, there is no prompting and the effect is
1033    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
1034    * {@link #disableTable(byte[])}
1035    *
1036    * @param pattern The pattern to match table names against
1037    * @return Table descriptors for tables that couldn't be disabled
1038    * @throws IOException
1039    */
1040   public HTableDescriptor[] disableTables(Pattern pattern) throws IOException {
1041     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
1042     for (HTableDescriptor table : listTables(pattern)) {
1043       if (isTableEnabled(table.getTableName())) {
1044         try {
1045           disableTable(table.getTableName());
1046         } catch (IOException ex) {
1047           LOG.info("Failed to disable table " + table.getTableName(), ex);
1048           failed.add(table);
1049         }
1050       }
1051     }
1052     return failed.toArray(new HTableDescriptor[failed.size()]);
1053   }
1054 
1055   /*
1056    * Checks whether table exists. If not, throws TableNotFoundException
1057    * @param tableName
1058    */
1059   private void checkTableExistence(TableName tableName) throws IOException {
1060     if (!tableExists(tableName)) {
1061       throw new TableNotFoundException(tableName);
1062     }
1063   }
1064 
1065   /**
1066    * @param tableName name of table to check
1067    * @return true if table is on-line
1068    * @throws IOException if a remote or network exception occurs
1069    */
1070   public boolean isTableEnabled(TableName tableName) throws IOException {
1071     checkTableExistence(tableName);
1072     return connection.isTableEnabled(tableName);
1073   }
1074 
1075   public boolean isTableEnabled(byte[] tableName) throws IOException {
1076     return isTableEnabled(TableName.valueOf(tableName));
1077   }
1078 
1079   public boolean isTableEnabled(String tableName) throws IOException {
1080     return isTableEnabled(TableName.valueOf(tableName));
1081   }
1082 
1083 
1084 
1085   /**
1086    * @param tableName name of table to check
1087    * @return true if table is off-line
1088    * @throws IOException if a remote or network exception occurs
1089    */
1090   public boolean isTableDisabled(TableName tableName) throws IOException {
1091     checkTableExistence(tableName);
1092     return connection.isTableDisabled(tableName);
1093   }
1094 
1095   public boolean isTableDisabled(byte[] tableName) throws IOException {
1096     return isTableDisabled(TableName.valueOf(tableName));
1097   }
1098 
1099   public boolean isTableDisabled(String tableName) throws IOException {
1100     return isTableDisabled(TableName.valueOf(tableName));
1101   }
1102 
1103   /**
1104    * @param tableName name of table to check
1105    * @return true if all regions of the table are available
1106    * @throws IOException if a remote or network exception occurs
1107    */
1108   public boolean isTableAvailable(TableName tableName) throws IOException {
1109     return connection.isTableAvailable(tableName);
1110   }
1111 
1112   public boolean isTableAvailable(byte[] tableName) throws IOException {
1113     return isTableAvailable(TableName.valueOf(tableName));
1114   }
1115 
1116   public boolean isTableAvailable(String tableName) throws IOException {
1117     return isTableAvailable(TableName.valueOf(tableName));
1118   }
1119 
1120   /**
1121    * Use this api to check if the table has been created with the specified number of
1122    * splitkeys which was used while creating the given table.
1123    * Note : If this api is used after a table's region gets splitted, the api may return
1124    * false.
1125    * @param tableName
1126    *          name of table to check
1127    * @param splitKeys
1128    *          keys to check if the table has been created with all split keys
1129    * @throws IOException
1130    *           if a remote or network excpetion occurs
1131    */
1132   public boolean isTableAvailable(TableName tableName,
1133                                   byte[][] splitKeys) throws IOException {
1134     return connection.isTableAvailable(tableName, splitKeys);
1135   }
1136 
1137   public boolean isTableAvailable(byte[] tableName,
1138                                   byte[][] splitKeys) throws IOException {
1139     return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1140   }
1141 
1142   public boolean isTableAvailable(String tableName,
1143                                   byte[][] splitKeys) throws IOException {
1144     return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1145   }
1146 
1147   /**
1148    * Get the status of alter command - indicates how many regions have received
1149    * the updated schema Asynchronous operation.
1150    *
1151    * @param tableName TableName instance
1152    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1153    *         regions that are yet to be updated Pair.getSecond() is the total number
1154    *         of regions of the table
1155    * @throws IOException
1156    *           if a remote or network exception occurs
1157    */
1158   public Pair<Integer, Integer> getAlterStatus(final TableName tableName)
1159   throws IOException {
1160     return executeCallable(new MasterCallable<Pair<Integer, Integer>>(getConnection()) {
1161       @Override
1162       public Pair<Integer, Integer> call() throws ServiceException {
1163         GetSchemaAlterStatusRequest req = RequestConverter
1164             .buildGetSchemaAlterStatusRequest(tableName);
1165         GetSchemaAlterStatusResponse ret = master.getSchemaAlterStatus(null, req);
1166         Pair<Integer, Integer> pair = new Pair<Integer, Integer>(Integer.valueOf(ret
1167             .getYetToUpdateRegions()), Integer.valueOf(ret.getTotalRegions()));
1168         return pair;
1169       }
1170     });
1171   }
1172 
1173   /**
1174    * Get the status of alter command - indicates how many regions have received
1175    * the updated schema Asynchronous operation.
1176    *
1177    * @param tableName
1178    *          name of the table to get the status of
1179    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1180    *         regions that are yet to be updated Pair.getSecond() is the total number
1181    *         of regions of the table
1182    * @throws IOException
1183    *           if a remote or network exception occurs
1184    */
1185   public Pair<Integer, Integer> getAlterStatus(final byte[] tableName)
1186    throws IOException {
1187     return getAlterStatus(TableName.valueOf(tableName));
1188   }
1189 
1190   /**
1191    * Add a column to an existing table.
1192    * Asynchronous operation.
1193    *
1194    * @param tableName name of the table to add column to
1195    * @param column column descriptor of column to be added
1196    * @throws IOException if a remote or network exception occurs
1197    */
1198   public void addColumn(final byte[] tableName, HColumnDescriptor column)
1199   throws IOException {
1200     addColumn(TableName.valueOf(tableName), column);
1201   }
1202 
1203 
1204   /**
1205    * Add a column to an existing table.
1206    * Asynchronous operation.
1207    *
1208    * @param tableName name of the table to add column to
1209    * @param column column descriptor of column to be added
1210    * @throws IOException if a remote or network exception occurs
1211    */
1212   public void addColumn(final String tableName, HColumnDescriptor column)
1213   throws IOException {
1214     addColumn(TableName.valueOf(tableName), column);
1215   }
1216 
1217   /**
1218    * Add a column to an existing table.
1219    * Asynchronous operation.
1220    *
1221    * @param tableName name of the table to add column to
1222    * @param column column descriptor of column to be added
1223    * @throws IOException if a remote or network exception occurs
1224    */
1225   public void addColumn(final TableName tableName, final HColumnDescriptor column)
1226   throws IOException {
1227     executeCallable(new MasterCallable<Void>(getConnection()) {
1228       @Override
1229       public Void call() throws ServiceException {
1230         AddColumnRequest req = RequestConverter.buildAddColumnRequest(tableName, column);
1231         master.addColumn(null,req);
1232         return null;
1233       }
1234     });
1235   }
1236 
1237   /**
1238    * Delete a column from a table.
1239    * Asynchronous operation.
1240    *
1241    * @param tableName name of table
1242    * @param columnName name of column to be deleted
1243    * @throws IOException if a remote or network exception occurs
1244    */
1245   public void deleteColumn(final byte[] tableName, final String columnName)
1246   throws IOException {
1247     deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
1248   }
1249 
1250   /**
1251    * Delete a column from a table.
1252    * Asynchronous operation.
1253    *
1254    * @param tableName name of table
1255    * @param columnName name of column to be deleted
1256    * @throws IOException if a remote or network exception occurs
1257    */
1258   public void deleteColumn(final String tableName, final String columnName)
1259   throws IOException {
1260     deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
1261   }
1262 
1263   /**
1264    * Delete a column from a table.
1265    * Asynchronous operation.
1266    *
1267    * @param tableName name of table
1268    * @param columnName name of column to be deleted
1269    * @throws IOException if a remote or network exception occurs
1270    */
1271   public void deleteColumn(final TableName tableName, final byte [] columnName)
1272   throws IOException {
1273     executeCallable(new MasterCallable<Void>(getConnection()) {
1274       @Override
1275       public Void call() throws ServiceException {
1276         DeleteColumnRequest req = RequestConverter.buildDeleteColumnRequest(tableName, columnName);
1277         master.deleteColumn(null,req);
1278         return null;
1279       }
1280     });
1281   }
1282 
1283   /**
1284    * Modify an existing column family on a table.
1285    * Asynchronous operation.
1286    *
1287    * @param tableName name of table
1288    * @param descriptor new column descriptor to use
1289    * @throws IOException if a remote or network exception occurs
1290    */
1291   public void modifyColumn(final String tableName, HColumnDescriptor descriptor)
1292   throws IOException {
1293     modifyColumn(TableName.valueOf(tableName), descriptor);
1294   }
1295 
1296   /**
1297    * Modify an existing column family on a table.
1298    * Asynchronous operation.
1299    *
1300    * @param tableName name of table
1301    * @param descriptor new column descriptor to use
1302    * @throws IOException if a remote or network exception occurs
1303    */
1304   public void modifyColumn(final byte[] tableName, HColumnDescriptor descriptor)
1305   throws IOException {
1306     modifyColumn(TableName.valueOf(tableName), descriptor);
1307   }
1308 
1309 
1310 
1311   /**
1312    * Modify an existing column family on a table.
1313    * Asynchronous operation.
1314    *
1315    * @param tableName name of table
1316    * @param descriptor new column descriptor to use
1317    * @throws IOException if a remote or network exception occurs
1318    */
1319   public void modifyColumn(final TableName tableName, final HColumnDescriptor descriptor)
1320   throws IOException {
1321     executeCallable(new MasterCallable<Void>(getConnection()) {
1322       @Override
1323       public Void call() throws ServiceException {
1324         ModifyColumnRequest req = RequestConverter.buildModifyColumnRequest(tableName, descriptor);
1325         master.modifyColumn(null,req);
1326         return null;
1327       }
1328     });
1329   }
1330 
1331   /**
1332    * Close a region. For expert-admins.  Runs close on the regionserver.  The
1333    * master will not be informed of the close.
1334    * @param regionname region name to close
1335    * @param serverName If supplied, we'll use this location rather than
1336    * the one currently in <code>hbase:meta</code>
1337    * @throws IOException if a remote or network exception occurs
1338    */
1339   public void closeRegion(final String regionname, final String serverName)
1340   throws IOException {
1341     closeRegion(Bytes.toBytes(regionname), serverName);
1342   }
1343 
1344   /**
1345    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1346    * master will not be informed of the close.
1347    * @param regionname region name to close
1348    * @param serverName The servername of the regionserver.  If passed null we
1349    * will use servername found in the hbase:meta table. A server name
1350    * is made of host, port and startcode.  Here is an example:
1351    * <code> host187.example.com,60020,1289493121758</code>
1352    * @throws IOException if a remote or network exception occurs
1353    */
1354   public void closeRegion(final byte [] regionname, final String serverName)
1355   throws IOException {
1356     CatalogTracker ct = getCatalogTracker();
1357     try {
1358       if (serverName != null) {
1359         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1360         if (pair == null || pair.getFirst() == null) {
1361           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1362         } else {
1363           closeRegion(ServerName.valueOf(serverName), pair.getFirst());
1364         }
1365       } else {
1366         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1367         if (pair == null) {
1368           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1369         } else if (pair.getSecond() == null) {
1370           throw new NoServerForRegionException(Bytes.toStringBinary(regionname));
1371         } else {
1372           closeRegion(pair.getSecond(), pair.getFirst());
1373         }
1374       }
1375     } finally {
1376       cleanupCatalogTracker(ct);
1377     }
1378   }
1379 
1380   /**
1381    * For expert-admins. Runs close on the regionserver. Closes a region based on
1382    * the encoded region name. The region server name is mandatory. If the
1383    * servername is provided then based on the online regions in the specified
1384    * regionserver the specified region will be closed. The master will not be
1385    * informed of the close. Note that the regionname is the encoded regionname.
1386    *
1387    * @param encodedRegionName
1388    *          The encoded region name; i.e. the hash that makes up the region
1389    *          name suffix: e.g. if regionname is
1390    *          <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>
1391    *          , then the encoded region name is:
1392    *          <code>527db22f95c8a9e0116f0cc13c680396</code>.
1393    * @param serverName
1394    *          The servername of the regionserver. A server name is made of host,
1395    *          port and startcode. This is mandatory. Here is an example:
1396    *          <code> host187.example.com,60020,1289493121758</code>
1397    * @return true if the region was closed, false if not.
1398    * @throws IOException
1399    *           if a remote or network exception occurs
1400    */
1401   public boolean closeRegionWithEncodedRegionName(final String encodedRegionName,
1402       final String serverName) throws IOException {
1403     if (null == serverName || ("").equals(serverName.trim())) {
1404       throw new IllegalArgumentException(
1405           "The servername cannot be null or empty.");
1406     }
1407     ServerName sn = ServerName.valueOf(serverName);
1408     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1409     // Close the region without updating zk state.
1410     CloseRegionRequest request =
1411       RequestConverter.buildCloseRegionRequest(sn, encodedRegionName, false);
1412     try {
1413       CloseRegionResponse response = admin.closeRegion(null, request);
1414       boolean isRegionClosed = response.getClosed();
1415       if (false == isRegionClosed) {
1416         LOG.error("Not able to close the region " + encodedRegionName + ".");
1417       }
1418       return isRegionClosed;
1419     } catch (ServiceException se) {
1420       throw ProtobufUtil.getRemoteException(se);
1421     }
1422   }
1423 
1424   /**
1425    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1426    * master will not be informed of the close.
1427    * @param sn
1428    * @param hri
1429    * @throws IOException
1430    */
1431   public void closeRegion(final ServerName sn, final HRegionInfo hri)
1432   throws IOException {
1433     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1434     // Close the region without updating zk state.
1435     ProtobufUtil.closeRegion(admin, sn, hri.getRegionName(), false);
1436   }
1437 
1438   /**
1439    * Get all the online regions on a region server.
1440    */
1441   public List<HRegionInfo> getOnlineRegions(
1442       final ServerName sn) throws IOException {
1443     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1444     return ProtobufUtil.getOnlineRegions(admin);
1445   }
1446 
1447   /**
1448    * Flush a table or an individual region.
1449    * Synchronous operation.
1450    *
1451    * @param tableNameOrRegionName table or region to flush
1452    * @throws IOException if a remote or network exception occurs
1453    * @throws InterruptedException
1454    */
1455   public void flush(final String tableNameOrRegionName)
1456   throws IOException, InterruptedException {
1457     flush(Bytes.toBytes(tableNameOrRegionName));
1458   }
1459 
1460   /**
1461    * Flush a table or an individual region.
1462    * Synchronous operation.
1463    *
1464    * @param tableNameOrRegionName table or region to flush
1465    * @throws IOException if a remote or network exception occurs
1466    * @throws InterruptedException
1467    */
1468   public void flush(final byte[] tableNameOrRegionName)
1469   throws IOException, InterruptedException {
1470     CatalogTracker ct = getCatalogTracker();
1471     try {
1472       Pair<HRegionInfo, ServerName> regionServerPair
1473         = getRegion(tableNameOrRegionName, ct);
1474       if (regionServerPair != null) {
1475         if (regionServerPair.getSecond() == null) {
1476           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1477         } else {
1478           flush(regionServerPair.getSecond(), regionServerPair.getFirst());
1479         }
1480       } else {
1481         final TableName tableName = checkTableExists(
1482             TableName.valueOf(tableNameOrRegionName), ct);
1483         List<Pair<HRegionInfo, ServerName>> pairs =
1484           MetaReader.getTableRegionsAndLocations(ct,
1485               tableName);
1486         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1487           if (pair.getFirst().isOffline()) continue;
1488           if (pair.getSecond() == null) continue;
1489           try {
1490             flush(pair.getSecond(), pair.getFirst());
1491           } catch (NotServingRegionException e) {
1492             if (LOG.isDebugEnabled()) {
1493               LOG.debug("Trying to flush " + pair.getFirst() + ": " +
1494                 StringUtils.stringifyException(e));
1495             }
1496           }
1497         }
1498       }
1499     } finally {
1500       cleanupCatalogTracker(ct);
1501     }
1502   }
1503 
1504   private void flush(final ServerName sn, final HRegionInfo hri)
1505   throws IOException {
1506     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1507     FlushRegionRequest request =
1508       RequestConverter.buildFlushRegionRequest(hri.getRegionName());
1509     try {
1510       admin.flushRegion(null, request);
1511     } catch (ServiceException se) {
1512       throw ProtobufUtil.getRemoteException(se);
1513     }
1514   }
1515 
1516   /**
1517    * Compact a table or an individual region.
1518    * Asynchronous operation.
1519    *
1520    * @param tableNameOrRegionName table or region to compact
1521    * @throws IOException if a remote or network exception occurs
1522    * @throws InterruptedException
1523    */
1524   public void compact(final String tableNameOrRegionName)
1525   throws IOException, InterruptedException {
1526     compact(Bytes.toBytes(tableNameOrRegionName));
1527   }
1528 
1529   /**
1530    * Compact a table or an individual region.
1531    * Asynchronous operation.
1532    *
1533    * @param tableNameOrRegionName table or region to compact
1534    * @throws IOException if a remote or network exception occurs
1535    * @throws InterruptedException
1536    */
1537   public void compact(final byte[] tableNameOrRegionName)
1538   throws IOException, InterruptedException {
1539     compact(tableNameOrRegionName, null, false);
1540   }
1541 
1542   /**
1543    * Compact a column family within a table or region.
1544    * Asynchronous operation.
1545    *
1546    * @param tableOrRegionName table or region to compact
1547    * @param columnFamily column family within a table or region
1548    * @throws IOException if a remote or network exception occurs
1549    * @throws InterruptedException
1550    */
1551   public void compact(String tableOrRegionName, String columnFamily)
1552     throws IOException,  InterruptedException {
1553     compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily));
1554   }
1555 
1556   /**
1557    * Compact a column family within a table or region.
1558    * Asynchronous operation.
1559    *
1560    * @param tableNameOrRegionName table or region to compact
1561    * @param columnFamily column family within a table or region
1562    * @throws IOException if a remote or network exception occurs
1563    * @throws InterruptedException
1564    */
1565   public void compact(final byte[] tableNameOrRegionName, final byte[] columnFamily)
1566   throws IOException, InterruptedException {
1567     compact(tableNameOrRegionName, columnFamily, false);
1568   }
1569 
1570   /**
1571    * Major compact a table or an individual region.
1572    * Asynchronous operation.
1573    *
1574    * @param tableNameOrRegionName table or region to major compact
1575    * @throws IOException if a remote or network exception occurs
1576    * @throws InterruptedException
1577    */
1578   public void majorCompact(final String tableNameOrRegionName)
1579   throws IOException, InterruptedException {
1580     majorCompact(Bytes.toBytes(tableNameOrRegionName));
1581   }
1582 
1583   /**
1584    * Major compact a table or an individual region.
1585    * Asynchronous operation.
1586    *
1587    * @param tableNameOrRegionName table or region to major compact
1588    * @throws IOException if a remote or network exception occurs
1589    * @throws InterruptedException
1590    */
1591   public void majorCompact(final byte[] tableNameOrRegionName)
1592   throws IOException, InterruptedException {
1593     compact(tableNameOrRegionName, null, true);
1594   }
1595 
1596   /**
1597    * Major compact a column family within a table or region.
1598    * Asynchronous operation.
1599    *
1600    * @param tableNameOrRegionName table or region to major compact
1601    * @param columnFamily column family within a table or region
1602    * @throws IOException if a remote or network exception occurs
1603    * @throws InterruptedException
1604    */
1605   public void majorCompact(final String tableNameOrRegionName,
1606     final String columnFamily) throws IOException, InterruptedException {
1607     majorCompact(Bytes.toBytes(tableNameOrRegionName),
1608       Bytes.toBytes(columnFamily));
1609   }
1610 
1611   /**
1612    * Major compact a column family within a table or region.
1613    * Asynchronous operation.
1614    *
1615    * @param tableNameOrRegionName table or region to major compact
1616    * @param columnFamily column family within a table or region
1617    * @throws IOException if a remote or network exception occurs
1618    * @throws InterruptedException
1619    */
1620   public void majorCompact(final byte[] tableNameOrRegionName,
1621     final byte[] columnFamily) throws IOException, InterruptedException {
1622     compact(tableNameOrRegionName, columnFamily, true);
1623   }
1624 
1625   /**
1626    * Compact a table or an individual region.
1627    * Asynchronous operation.
1628    *
1629    * @param tableNameOrRegionName table or region to compact
1630    * @param columnFamily column family within a table or region
1631    * @param major True if we are to do a major compaction.
1632    * @throws IOException if a remote or network exception occurs
1633    * @throws InterruptedException
1634    */
1635   private void compact(final byte[] tableNameOrRegionName,
1636     final byte[] columnFamily,final boolean major)
1637   throws IOException, InterruptedException {
1638     CatalogTracker ct = getCatalogTracker();
1639     try {
1640       Pair<HRegionInfo, ServerName> regionServerPair
1641         = getRegion(tableNameOrRegionName, ct);
1642       if (regionServerPair != null) {
1643         if (regionServerPair.getSecond() == null) {
1644           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1645         } else {
1646           compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
1647         }
1648       } else {
1649         final TableName tableName =
1650             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
1651         List<Pair<HRegionInfo, ServerName>> pairs =
1652           MetaReader.getTableRegionsAndLocations(ct,
1653               tableName);
1654         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1655           if (pair.getFirst().isOffline()) continue;
1656           if (pair.getSecond() == null) continue;
1657           try {
1658             compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
1659           } catch (NotServingRegionException e) {
1660             if (LOG.isDebugEnabled()) {
1661               LOG.debug("Trying to" + (major ? " major" : "") + " compact " +
1662                 pair.getFirst() + ": " +
1663                 StringUtils.stringifyException(e));
1664             }
1665           }
1666         }
1667       }
1668     } finally {
1669       cleanupCatalogTracker(ct);
1670     }
1671   }
1672 
1673   private void compact(final ServerName sn, final HRegionInfo hri,
1674       final boolean major, final byte [] family)
1675   throws IOException {
1676     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1677     CompactRegionRequest request =
1678       RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major, family);
1679     try {
1680       admin.compactRegion(null, request);
1681     } catch (ServiceException se) {
1682       throw ProtobufUtil.getRemoteException(se);
1683     }
1684   }
1685 
1686   /**
1687    * Compact all regions on the region server
1688    * @param regionserver the region server name
1689    * @param major if it's major compaction
1690    * @throws IOException
1691    * @throws InterruptedException
1692    */
1693   public void compactRegionServer(final ServerName sn, boolean major)
1694       throws IOException, InterruptedException {
1695     for (HRegionInfo region : getOnlineRegions(sn)) {
1696       compact(sn, region, major, null);
1697     }
1698   }
1699 
1700   /**
1701    * Move the region <code>r</code> to <code>dest</code>.
1702    * @param encodedRegionName The encoded region name; i.e. the hash that makes
1703    * up the region name suffix: e.g. if regionname is
1704    * <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>,
1705    * then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>.
1706    * @param destServerName The servername of the destination regionserver.  If
1707    * passed the empty byte array we'll assign to a random server.  A server name
1708    * is made of host, port and startcode.  Here is an example:
1709    * <code> host187.example.com,60020,1289493121758</code>
1710    * @throws UnknownRegionException Thrown if we can't find a region named
1711    * <code>encodedRegionName</code>
1712    * @throws ZooKeeperConnectionException
1713    * @throws MasterNotRunningException
1714    */
1715   public void move(final byte [] encodedRegionName, final byte [] destServerName)
1716   throws HBaseIOException, MasterNotRunningException, ZooKeeperConnectionException {
1717     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1718     try {
1719       MoveRegionRequest request =
1720         RequestConverter.buildMoveRegionRequest(encodedRegionName, destServerName);
1721       stub.moveRegion(null,request);
1722     } catch (ServiceException se) {
1723       IOException ioe = ProtobufUtil.getRemoteException(se);
1724       if (ioe instanceof HBaseIOException) {
1725         throw (HBaseIOException)ioe;
1726       }
1727       LOG.error("Unexpected exception: " + se + " from calling HMaster.moveRegion");
1728     } catch (DeserializationException de) {
1729       LOG.error("Could not parse destination server name: " + de);
1730     } finally {
1731       stub.close();
1732     }
1733   }
1734 
1735   /**
1736    * @param regionName
1737    *          Region name to assign.
1738    * @throws MasterNotRunningException
1739    * @throws ZooKeeperConnectionException
1740    * @throws IOException
1741    */
1742   public void assign(final byte[] regionName) throws MasterNotRunningException,
1743       ZooKeeperConnectionException, IOException {
1744     final byte[] toBeAssigned = getRegionName(regionName);
1745     executeCallable(new MasterCallable<Void>(getConnection()) {
1746       @Override
1747       public Void call() throws ServiceException {
1748         AssignRegionRequest request =
1749           RequestConverter.buildAssignRegionRequest(toBeAssigned);
1750         master.assignRegion(null,request);
1751         return null;
1752       }
1753     });
1754   }
1755 
1756   /**
1757    * Unassign a region from current hosting regionserver.  Region will then be
1758    * assigned to a regionserver chosen at random.  Region could be reassigned
1759    * back to the same server.  Use {@link #move(byte[], byte[])} if you want
1760    * to control the region movement.
1761    * @param regionName Region to unassign. Will clear any existing RegionPlan
1762    * if one found.
1763    * @param force If true, force unassign (Will remove region from
1764    * regions-in-transition too if present. If results in double assignment
1765    * use hbck -fix to resolve. To be used by experts).
1766    * @throws MasterNotRunningException
1767    * @throws ZooKeeperConnectionException
1768    * @throws IOException
1769    */
1770   public void unassign(final byte [] regionName, final boolean force)
1771   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
1772     final byte[] toBeUnassigned = getRegionName(regionName);
1773     executeCallable(new MasterCallable<Void>(getConnection()) {
1774       @Override
1775       public Void call() throws ServiceException {
1776         UnassignRegionRequest request =
1777           RequestConverter.buildUnassignRegionRequest(toBeUnassigned, force);
1778         master.unassignRegion(null,request);
1779         return null;
1780       }
1781     });
1782   }
1783 
1784   /**
1785    * Offline specified region from master's in-memory state. It will not attempt to reassign the
1786    * region as in unassign. This API can be used when a region not served by any region server and
1787    * still online as per Master's in memory state. If this API is incorrectly used on active region
1788    * then master will loose track of that region.
1789    *
1790    * This is a special method that should be used by experts or hbck.
1791    *
1792    * @param regionName
1793    *          Region to offline.
1794    * @throws IOException
1795    */
1796   public void offline(final byte [] regionName)
1797   throws IOException {
1798     MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
1799     try {
1800       master.offlineRegion(null,RequestConverter.buildOfflineRegionRequest(regionName));
1801     } catch (ServiceException se) {
1802       throw ProtobufUtil.getRemoteException(se);
1803     } finally {
1804       master.close();
1805     }
1806   }
1807 
1808   /**
1809    * Turn the load balancer on or off.
1810    * @param on If true, enable balancer. If false, disable balancer.
1811    * @param synchronous If true, it waits until current balance() call, if outstanding, to return.
1812    * @return Previous balancer value
1813    */
1814   public boolean setBalancerRunning(final boolean on, final boolean synchronous)
1815   throws MasterNotRunningException, ZooKeeperConnectionException {
1816     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1817     try {
1818       SetBalancerRunningRequest req =
1819         RequestConverter.buildSetBalancerRunningRequest(on, synchronous);
1820       return stub.setBalancerRunning(null, req).getPrevBalanceValue();
1821     } catch (ServiceException se) {
1822       IOException ioe = ProtobufUtil.getRemoteException(se);
1823       if (ioe instanceof MasterNotRunningException) {
1824         throw (MasterNotRunningException)ioe;
1825       }
1826       if (ioe instanceof ZooKeeperConnectionException) {
1827         throw (ZooKeeperConnectionException)ioe;
1828       }
1829 
1830       // Throwing MasterNotRunningException even though not really valid in order to not
1831       // break interface by adding additional exception type.
1832       throw new MasterNotRunningException("Unexpected exception when calling balanceSwitch",se);
1833     } finally {
1834       stub.close();
1835     }
1836   }
1837 
1838   /**
1839    * Invoke the balancer.  Will run the balancer and if regions to move, it will
1840    * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
1841    * logs.
1842    * @return True if balancer ran, false otherwise.
1843    */
1844   public boolean balancer()
1845   throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException {
1846     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1847     try {
1848       return stub.balance(null,RequestConverter.buildBalanceRequest()).getBalancerRan();
1849     } finally {
1850       stub.close();
1851     }
1852   }
1853 
1854   /**
1855    * Enable/Disable the catalog janitor
1856    * @param enable if true enables the catalog janitor
1857    * @return the previous state
1858    * @throws ServiceException
1859    * @throws MasterNotRunningException
1860    */
1861   public boolean enableCatalogJanitor(boolean enable)
1862       throws ServiceException, MasterNotRunningException {
1863     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1864     try {
1865       return stub.enableCatalogJanitor(null,
1866           RequestConverter.buildEnableCatalogJanitorRequest(enable)).getPrevValue();
1867     } finally {
1868       stub.close();
1869     }
1870   }
1871 
1872   /**
1873    * Ask for a scan of the catalog table
1874    * @return the number of entries cleaned
1875    * @throws ServiceException
1876    * @throws MasterNotRunningException
1877    */
1878   public int runCatalogScan() throws ServiceException, MasterNotRunningException {
1879     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1880     try {
1881       return stub.runCatalogScan(null,
1882           RequestConverter.buildCatalogScanRequest()).getScanResult();
1883     } finally {
1884       stub.close();
1885     }
1886   }
1887 
1888   /**
1889    * Query on the catalog janitor state (Enabled/Disabled?)
1890    * @throws ServiceException
1891    * @throws org.apache.hadoop.hbase.MasterNotRunningException
1892    */
1893   public boolean isCatalogJanitorEnabled() throws ServiceException, MasterNotRunningException {
1894     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1895     try {
1896       return stub.isCatalogJanitorEnabled(null,
1897           RequestConverter.buildIsCatalogJanitorEnabledRequest()).getValue();
1898     } finally {
1899       stub.close();
1900     }
1901   }
1902 
1903   /**
1904    * Merge two regions. Asynchronous operation.
1905    * @param encodedNameOfRegionA encoded name of region a
1906    * @param encodedNameOfRegionB encoded name of region b
1907    * @param forcible true if do a compulsory merge, otherwise we will only merge
1908    *          two adjacent regions
1909    * @throws IOException
1910    */
1911   public void mergeRegions(final byte[] encodedNameOfRegionA,
1912       final byte[] encodedNameOfRegionB, final boolean forcible)
1913       throws IOException {
1914     MasterKeepAliveConnection master = connection
1915         .getKeepAliveMasterService();
1916     try {
1917       DispatchMergingRegionsRequest request = RequestConverter
1918           .buildDispatchMergingRegionsRequest(encodedNameOfRegionA,
1919               encodedNameOfRegionB, forcible);
1920       master.dispatchMergingRegions(null, request);
1921     } catch (ServiceException se) {
1922       IOException ioe = ProtobufUtil.getRemoteException(se);
1923       if (ioe instanceof UnknownRegionException) {
1924         throw (UnknownRegionException) ioe;
1925       }
1926       if (ioe instanceof MergeRegionException) {
1927         throw (MergeRegionException) ioe;
1928       }
1929       LOG.error("Unexpected exception: " + se
1930           + " from calling HMaster.dispatchMergingRegions");
1931     } catch (DeserializationException de) {
1932       LOG.error("Could not parse destination server name: " + de);
1933     } finally {
1934       master.close();
1935     }
1936   }
1937 
1938   /**
1939    * Split a table or an individual region.
1940    * Asynchronous operation.
1941    *
1942    * @param tableNameOrRegionName table or region to split
1943    * @throws IOException if a remote or network exception occurs
1944    * @throws InterruptedException
1945    */
1946   public void split(final String tableNameOrRegionName)
1947   throws IOException, InterruptedException {
1948     split(Bytes.toBytes(tableNameOrRegionName));
1949   }
1950 
1951   /**
1952    * Split a table or an individual region.  Implicitly finds an optimal split
1953    * point.  Asynchronous operation.
1954    *
1955    * @param tableNameOrRegionName table to region to split
1956    * @throws IOException if a remote or network exception occurs
1957    * @throws InterruptedException
1958    */
1959   public void split(final byte[] tableNameOrRegionName)
1960   throws IOException, InterruptedException {
1961     split(tableNameOrRegionName, null);
1962   }
1963 
1964   public void split(final String tableNameOrRegionName,
1965     final String splitPoint) throws IOException, InterruptedException {
1966     split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint));
1967   }
1968 
1969   /**
1970    * Split a table or an individual region.
1971    * Asynchronous operation.
1972    *
1973    * @param tableNameOrRegionName table to region to split
1974    * @param splitPoint the explicit position to split on
1975    * @throws IOException if a remote or network exception occurs
1976    * @throws InterruptedException interrupt exception occurred
1977    */
1978   public void split(final byte[] tableNameOrRegionName,
1979       final byte [] splitPoint) throws IOException, InterruptedException {
1980     CatalogTracker ct = getCatalogTracker();
1981     try {
1982       Pair<HRegionInfo, ServerName> regionServerPair
1983         = getRegion(tableNameOrRegionName, ct);
1984       if (regionServerPair != null) {
1985         if (regionServerPair.getSecond() == null) {
1986             throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1987         } else {
1988           split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
1989         }
1990       } else {
1991         final TableName tableName =
1992             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
1993         List<Pair<HRegionInfo, ServerName>> pairs =
1994           MetaReader.getTableRegionsAndLocations(ct,
1995               tableName);
1996         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1997           // May not be a server for a particular row
1998           if (pair.getSecond() == null) continue;
1999           HRegionInfo r = pair.getFirst();
2000           // check for parents
2001           if (r.isSplitParent()) continue;
2002           // if a split point given, only split that particular region
2003           if (splitPoint != null && !r.containsRow(splitPoint)) continue;
2004           // call out to region server to do split now
2005           split(pair.getSecond(), pair.getFirst(), splitPoint);
2006         }
2007       }
2008     } finally {
2009       cleanupCatalogTracker(ct);
2010     }
2011   }
2012 
2013   private void split(final ServerName sn, final HRegionInfo hri,
2014       byte[] splitPoint) throws IOException {
2015     if (hri.getStartKey() != null && splitPoint != null &&
2016          Bytes.compareTo(hri.getStartKey(), splitPoint) == 0) {
2017        throw new IOException("should not give a splitkey which equals to startkey!");
2018     }
2019     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2020     ProtobufUtil.split(admin, hri, splitPoint);
2021   }
2022 
2023   /**
2024    * Modify an existing table, more IRB friendly version.
2025    * Asynchronous operation.  This means that it may be a while before your
2026    * schema change is updated across all of the table.
2027    *
2028    * @param tableName name of table.
2029    * @param htd modified description of the table
2030    * @throws IOException if a remote or network exception occurs
2031    */
2032   public void modifyTable(final TableName tableName, final HTableDescriptor htd)
2033   throws IOException {
2034     if (!tableName.equals(htd.getTableName())) {
2035       throw new IllegalArgumentException("the specified table name '" + tableName +
2036         "' doesn't match with the HTD one: " + htd.getTableName());
2037     }
2038 
2039     executeCallable(new MasterCallable<Void>(getConnection()) {
2040       @Override
2041       public Void call() throws ServiceException {
2042         ModifyTableRequest request = RequestConverter.buildModifyTableRequest(tableName, htd);
2043         master.modifyTable(null, request);
2044         return null;
2045       }
2046     });
2047   }
2048 
2049   public void modifyTable(final byte[] tableName, final HTableDescriptor htd)
2050   throws IOException {
2051     modifyTable(TableName.valueOf(tableName), htd);
2052   }
2053 
2054   public void modifyTable(final String tableName, final HTableDescriptor htd)
2055   throws IOException {
2056     modifyTable(TableName.valueOf(tableName), htd);
2057   }
2058 
2059   /**
2060    * @param tableNameOrRegionName Name of a table or name of a region.
2061    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
2062    * @return a pair of HRegionInfo and ServerName if <code>tableNameOrRegionName</code> is
2063    *  a verified region name (we call {@link  MetaReader#getRegion( CatalogTracker, byte[])}
2064    *  else null.
2065    * Throw an exception if <code>tableNameOrRegionName</code> is null.
2066    * @throws IOException
2067    */
2068   Pair<HRegionInfo, ServerName> getRegion(final byte[] tableNameOrRegionName,
2069       final CatalogTracker ct) throws IOException {
2070     if (tableNameOrRegionName == null) {
2071       throw new IllegalArgumentException("Pass a table name or region name");
2072     }
2073     Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, tableNameOrRegionName);
2074     if (pair == null) {
2075       final AtomicReference<Pair<HRegionInfo, ServerName>> result =
2076         new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
2077       final String encodedName = Bytes.toString(tableNameOrRegionName);
2078       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2079         @Override
2080         public boolean processRow(Result data) throws IOException {
2081           HRegionInfo info = HRegionInfo.getHRegionInfo(data);
2082           if (info == null) {
2083             LOG.warn("No serialized HRegionInfo in " + data);
2084             return true;
2085           }
2086           if (!encodedName.equals(info.getEncodedName())) return true;
2087           ServerName sn = HRegionInfo.getServerName(data);
2088           result.set(new Pair<HRegionInfo, ServerName>(info, sn));
2089           return false; // found the region, stop
2090         }
2091       };
2092 
2093       MetaScanner.metaScan(conf, connection, visitor, null);
2094       pair = result.get();
2095     }
2096     return pair;
2097   }
2098 
2099   /**
2100    * If the input is a region name, it is returned as is. If it's an
2101    * encoded region name, the corresponding region is found from meta
2102    * and its region name is returned. If we can't find any region in
2103    * meta matching the input as either region name or encoded region
2104    * name, the input is returned as is. We don't throw unknown
2105    * region exception.
2106    */
2107   private byte[] getRegionName(
2108       final byte[] regionNameOrEncodedRegionName) throws IOException {
2109     if (Bytes.equals(regionNameOrEncodedRegionName,
2110         HRegionInfo.FIRST_META_REGIONINFO.getRegionName())
2111           || Bytes.equals(regionNameOrEncodedRegionName,
2112             HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes())) {
2113       return HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
2114     }
2115     CatalogTracker ct = getCatalogTracker();
2116     byte[] tmp = regionNameOrEncodedRegionName;
2117     try {
2118       Pair<HRegionInfo, ServerName> regionServerPair
2119         = getRegion(regionNameOrEncodedRegionName, ct);
2120       if (regionServerPair != null && regionServerPair.getFirst() != null) {
2121         tmp = regionServerPair.getFirst().getRegionName();
2122       }
2123     } finally {
2124       cleanupCatalogTracker(ct);
2125     }
2126     return tmp;
2127   }
2128 
2129   /**
2130    * Check if table exists or not
2131    * @param tableName Name of a table.
2132    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
2133    * @return tableName instance
2134    * @throws IOException if a remote or network exception occurs.
2135    * @throws TableNotFoundException if table does not exist.
2136    */
2137   //TODO rename this method
2138   private TableName checkTableExists(
2139       final TableName tableName, CatalogTracker ct)
2140       throws IOException {
2141     if (!MetaReader.tableExists(ct, tableName)) {
2142       throw new TableNotFoundException(tableName);
2143     }
2144     return tableName;
2145   }
2146 
2147   /**
2148    * Shuts down the HBase cluster
2149    * @throws IOException if a remote or network exception occurs
2150    */
2151   public synchronized void shutdown() throws IOException {
2152     executeCallable(new MasterCallable<Void>(getConnection()) {
2153       @Override
2154       public Void call() throws ServiceException {
2155         master.shutdown(null,ShutdownRequest.newBuilder().build());
2156         return null;
2157       }
2158     });
2159   }
2160 
2161   /**
2162    * Shuts down the current HBase master only.
2163    * Does not shutdown the cluster.
2164    * @see #shutdown()
2165    * @throws IOException if a remote or network exception occurs
2166    */
2167   public synchronized void stopMaster() throws IOException {
2168     executeCallable(new MasterCallable<Void>(getConnection()) {
2169       @Override
2170       public Void call() throws ServiceException {
2171         master.stopMaster(null,StopMasterRequest.newBuilder().build());
2172         return null;
2173       }
2174     });
2175   }
2176 
2177   /**
2178    * Stop the designated regionserver
2179    * @param hostnamePort Hostname and port delimited by a <code>:</code> as in
2180    * <code>example.org:1234</code>
2181    * @throws IOException if a remote or network exception occurs
2182    */
2183   public synchronized void stopRegionServer(final String hostnamePort)
2184   throws IOException {
2185     String hostname = Addressing.parseHostname(hostnamePort);
2186     int port = Addressing.parsePort(hostnamePort);
2187     AdminService.BlockingInterface admin =
2188       this.connection.getAdmin(ServerName.valueOf(hostname, port, 0));
2189     StopServerRequest request = RequestConverter.buildStopServerRequest(
2190       "Called by admin client " + this.connection.toString());
2191     try {
2192       admin.stopServer(null, request);
2193     } catch (ServiceException se) {
2194       throw ProtobufUtil.getRemoteException(se);
2195     }
2196   }
2197 
2198 
2199   /**
2200    * @return cluster status
2201    * @throws IOException if a remote or network exception occurs
2202    */
2203   public ClusterStatus getClusterStatus() throws IOException {
2204     return executeCallable(new MasterCallable<ClusterStatus>(getConnection()) {
2205       @Override
2206       public ClusterStatus call() throws ServiceException {
2207         GetClusterStatusRequest req = RequestConverter.buildGetClusterStatusRequest();
2208         return ClusterStatus.convert(master.getClusterStatus(null,req).getClusterStatus());
2209       }
2210     });
2211   }
2212 
2213   private HRegionLocation getFirstMetaServerForTable(final TableName tableName)
2214   throws IOException {
2215     return connection.locateRegion(TableName.META_TABLE_NAME,
2216       HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
2217   }
2218 
2219   /**
2220    * @return Configuration used by the instance.
2221    */
2222   public Configuration getConfiguration() {
2223     return this.conf;
2224   }
2225 
2226   /**
2227    * Create a new namespace
2228    * @param descriptor descriptor which describes the new namespace
2229    * @throws IOException
2230    */
2231   public void createNamespace(final NamespaceDescriptor descriptor) throws IOException {
2232     executeCallable(new MasterCallable<Void>(getConnection()) {
2233       @Override
2234       public Void call() throws Exception {
2235         master.createNamespace(null,
2236           CreateNamespaceRequest.newBuilder()
2237                 .setNamespaceDescriptor(ProtobufUtil
2238                     .toProtoNamespaceDescriptor(descriptor)).build());
2239         return null;
2240       }
2241     });
2242   }
2243 
2244   /**
2245    * Modify an existing namespace
2246    * @param descriptor descriptor which describes the new namespace
2247    * @throws IOException
2248    */
2249   public void modifyNamespace(final NamespaceDescriptor descriptor) throws IOException {
2250     executeCallable(new MasterCallable<Void>(getConnection()) {
2251       @Override
2252       public Void call() throws Exception {
2253         master.modifyNamespace(null, ModifyNamespaceRequest.newBuilder().
2254           setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(descriptor)).build());
2255         return null;
2256       }
2257     });
2258   }
2259 
2260   /**
2261    * Delete an existing namespace. Only empty namespaces (no tables) can be removed.
2262    * @param name namespace name
2263    * @throws IOException
2264    */
2265   public void deleteNamespace(final String name) throws IOException {
2266     executeCallable(new MasterCallable<Void>(getConnection()) {
2267       @Override
2268       public Void call() throws Exception {
2269         master.deleteNamespace(null, DeleteNamespaceRequest.newBuilder().
2270           setNamespaceName(name).build());
2271         return null;
2272       }
2273     });
2274   }
2275 
2276   /**
2277    * Get a namespace descriptor by name
2278    * @param name name of namespace descriptor
2279    * @return A descriptor
2280    * @throws IOException
2281    */
2282   public NamespaceDescriptor getNamespaceDescriptor(final String name) throws IOException {
2283     return
2284         executeCallable(new MasterCallable<NamespaceDescriptor>(getConnection()) {
2285           @Override
2286           public NamespaceDescriptor call() throws Exception {
2287             return ProtobufUtil.toNamespaceDescriptor(
2288               master.getNamespaceDescriptor(null, GetNamespaceDescriptorRequest.newBuilder().
2289                 setNamespaceName(name).build()).getNamespaceDescriptor());
2290           }
2291         });
2292   }
2293 
2294   /**
2295    * List available namespace descriptors
2296    * @return List of descriptors
2297    * @throws IOException
2298    */
2299   public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException {
2300     return
2301         executeCallable(new MasterCallable<NamespaceDescriptor[]>(getConnection()) {
2302           @Override
2303           public NamespaceDescriptor[] call() throws Exception {
2304             List<HBaseProtos.NamespaceDescriptor> list =
2305               master.listNamespaceDescriptors(null, ListNamespaceDescriptorsRequest.newBuilder().
2306                 build()).getNamespaceDescriptorList();
2307             NamespaceDescriptor[] res = new NamespaceDescriptor[list.size()];
2308             for(int i = 0; i < list.size(); i++) {
2309               res[i] = ProtobufUtil.toNamespaceDescriptor(list.get(i));
2310             }
2311             return res;
2312           }
2313         });
2314   }
2315 
2316   /**
2317    * Get list of table descriptors by namespace
2318    * @param name namespace name
2319    * @return A descriptor
2320    * @throws IOException
2321    */
2322   public HTableDescriptor[] listTableDescriptorsByNamespace(final String name) throws IOException {
2323     return
2324         executeCallable(new MasterCallable<HTableDescriptor[]>(getConnection()) {
2325           @Override
2326           public HTableDescriptor[] call() throws Exception {
2327             List<TableSchema> list =
2328               master.listTableDescriptorsByNamespace(null, ListTableDescriptorsByNamespaceRequest.
2329                 newBuilder().setNamespaceName(name).build()).getTableSchemaList();
2330             HTableDescriptor[] res = new HTableDescriptor[list.size()];
2331             for(int i=0; i < list.size(); i++) {
2332 
2333               res[i] = HTableDescriptor.convert(list.get(i));
2334             }
2335             return res;
2336           }
2337         });
2338   }
2339 
2340   /**
2341    * Get list of table names by namespace
2342    * @param name namespace name
2343    * @return The list of table names in the namespace
2344    * @throws IOException
2345    */
2346   public TableName[] listTableNamesByNamespace(final String name) throws IOException {
2347     return
2348         executeCallable(new MasterCallable<TableName[]>(getConnection()) {
2349           @Override
2350           public TableName[] call() throws Exception {
2351             List<HBaseProtos.TableName> tableNames =
2352               master.listTableNamesByNamespace(null, ListTableNamesByNamespaceRequest.
2353                 newBuilder().setNamespaceName(name).build())
2354                 .getTableNameList();
2355             TableName[] result = new TableName[tableNames.size()];
2356             for (int i = 0; i < tableNames.size(); i++) {
2357               result[i] = ProtobufUtil.toTableName(tableNames.get(i));
2358             }
2359             return result;
2360           }
2361         });
2362   }
2363 
2364   /**
2365    * Check to see if HBase is running. Throw an exception if not.
2366    * We consider that HBase is running if ZooKeeper and Master are running.
2367    *
2368    * @param conf system configuration
2369    * @throws MasterNotRunningException if the master is not running
2370    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
2371    */
2372   public static void checkHBaseAvailable(Configuration conf)
2373     throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException {
2374     Configuration copyOfConf = HBaseConfiguration.create(conf);
2375 
2376     // We set it to make it fail as soon as possible if HBase is not available
2377     copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
2378     copyOfConf.setInt("zookeeper.recovery.retry", 0);
2379 
2380     HConnectionManager.HConnectionImplementation connection
2381       = (HConnectionManager.HConnectionImplementation)
2382       HConnectionManager.getConnection(copyOfConf);
2383 
2384     try {
2385       // Check ZK first.
2386       // If the connection exists, we may have a connection to ZK that does
2387       //  not work anymore
2388       ZooKeeperKeepAliveConnection zkw = null;
2389       try {
2390         zkw = connection.getKeepAliveZooKeeperWatcher();
2391         zkw.getRecoverableZooKeeper().getZooKeeper().exists(
2392           zkw.baseZNode, false);
2393 
2394       } catch (IOException e) {
2395         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
2396       } catch (InterruptedException e) {
2397         throw (InterruptedIOException)
2398             new InterruptedIOException("Can't connect to ZooKeeper").initCause(e);
2399       } catch (KeeperException e) {
2400         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
2401       } finally {
2402         if (zkw != null) {
2403           zkw.close();
2404         }
2405       }
2406 
2407       // Check Master
2408       connection.isMasterRunning();
2409 
2410     } finally {
2411       connection.close();
2412     }
2413   }
2414 
2415   /**
2416    * get the regions of a given table.
2417    *
2418    * @param tableName the name of the table
2419    * @return Ordered list of {@link HRegionInfo}.
2420    * @throws IOException
2421    */
2422   public List<HRegionInfo> getTableRegions(final TableName tableName)
2423   throws IOException {
2424     CatalogTracker ct = getCatalogTracker();
2425     List<HRegionInfo> Regions = null;
2426     try {
2427       Regions = MetaReader.getTableRegions(ct, tableName, true);
2428     } finally {
2429       cleanupCatalogTracker(ct);
2430     }
2431     return Regions;
2432   }
2433 
2434   public List<HRegionInfo> getTableRegions(final byte[] tableName)
2435   throws IOException {
2436     return getTableRegions(TableName.valueOf(tableName));
2437   }
2438 
2439   @Override
2440   public synchronized void close() throws IOException {
2441     if (cleanupConnectionOnClose && this.connection != null && !this.closed) {
2442       this.connection.close();
2443       this.closed = true;
2444     }
2445   }
2446 
2447   /**
2448    * Get tableDescriptors
2449    * @param tableNames List of table names
2450    * @return HTD[] the tableDescriptor
2451    * @throws IOException if a remote or network exception occurs
2452    */
2453   public HTableDescriptor[] getTableDescriptorsByTableName(List<TableName> tableNames)
2454   throws IOException {
2455     return this.connection.getHTableDescriptorsByTableName(tableNames);
2456   }
2457 
2458   /**
2459    * Get tableDescriptors
2460    * @param names List of table names
2461    * @return HTD[] the tableDescriptor
2462    * @throws IOException if a remote or network exception occurs
2463    */
2464   public HTableDescriptor[] getTableDescriptors(List<String> names)
2465   throws IOException {
2466     List<TableName> tableNames = new ArrayList<TableName>(names.size());
2467     for(String name : names) {
2468       tableNames.add(TableName.valueOf(name));
2469     }
2470     return getTableDescriptorsByTableName(tableNames);
2471   }
2472 
2473   /**
2474    * Roll the log writer. That is, start writing log messages to a new file.
2475    *
2476    * @param serverName
2477    *          The servername of the regionserver. A server name is made of host,
2478    *          port and startcode. This is mandatory. Here is an example:
2479    *          <code> host187.example.com,60020,1289493121758</code>
2480    * @return If lots of logs, flush the returned regions so next time through
2481    * we can clean logs. Returns null if nothing to flush.  Names are actual
2482    * region names as returned by {@link HRegionInfo#getEncodedName()}
2483    * @throws IOException if a remote or network exception occurs
2484    * @throws FailedLogCloseException
2485    */
2486  public synchronized  byte[][] rollHLogWriter(String serverName)
2487       throws IOException, FailedLogCloseException {
2488     ServerName sn = ServerName.valueOf(serverName);
2489     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2490     RollWALWriterRequest request = RequestConverter.buildRollWALWriterRequest();
2491     try {
2492       RollWALWriterResponse response = admin.rollWALWriter(null, request);
2493       int regionCount = response.getRegionToFlushCount();
2494       byte[][] regionsToFlush = new byte[regionCount][];
2495       for (int i = 0; i < regionCount; i++) {
2496         ByteString region = response.getRegionToFlush(i);
2497         regionsToFlush[i] = region.toByteArray();
2498       }
2499       return regionsToFlush;
2500     } catch (ServiceException se) {
2501       throw ProtobufUtil.getRemoteException(se);
2502     }
2503   }
2504 
2505   public String[] getMasterCoprocessors() {
2506     try {
2507       return getClusterStatus().getMasterCoprocessors();
2508     } catch (IOException e) {
2509       LOG.error("Could not getClusterStatus()",e);
2510       return null;
2511     }
2512   }
2513 
2514   /**
2515    * Get the current compaction state of a table or region.
2516    * It could be in a major compaction, a minor compaction, both, or none.
2517    *
2518    * @param tableNameOrRegionName table or region to major compact
2519    * @throws IOException if a remote or network exception occurs
2520    * @throws InterruptedException
2521    * @return the current compaction state
2522    */
2523   public CompactionState getCompactionState(final String tableNameOrRegionName)
2524       throws IOException, InterruptedException {
2525     return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
2526   }
2527 
2528   /**
2529    * Get the current compaction state of a table or region.
2530    * It could be in a major compaction, a minor compaction, both, or none.
2531    *
2532    * @param tableNameOrRegionName table or region to major compact
2533    * @throws IOException if a remote or network exception occurs
2534    * @throws InterruptedException
2535    * @return the current compaction state
2536    */
2537   public CompactionState getCompactionState(final byte[] tableNameOrRegionName)
2538       throws IOException, InterruptedException {
2539     CompactionState state = CompactionState.NONE;
2540     CatalogTracker ct = getCatalogTracker();
2541     try {
2542       Pair<HRegionInfo, ServerName> regionServerPair
2543         = getRegion(tableNameOrRegionName, ct);
2544       if (regionServerPair != null) {
2545         if (regionServerPair.getSecond() == null) {
2546           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
2547         } else {
2548           ServerName sn = regionServerPair.getSecond();
2549           AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2550           GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2551             regionServerPair.getFirst().getRegionName(), true);
2552           GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2553           return response.getCompactionState();
2554         }
2555       } else {
2556         final TableName tableName =
2557             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
2558         List<Pair<HRegionInfo, ServerName>> pairs =
2559           MetaReader.getTableRegionsAndLocations(ct, tableName);
2560         for (Pair<HRegionInfo, ServerName> pair: pairs) {
2561           if (pair.getFirst().isOffline()) continue;
2562           if (pair.getSecond() == null) continue;
2563           try {
2564             ServerName sn = pair.getSecond();
2565             AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2566             GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2567               pair.getFirst().getRegionName(), true);
2568             GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2569             switch (response.getCompactionState()) {
2570             case MAJOR_AND_MINOR:
2571               return CompactionState.MAJOR_AND_MINOR;
2572             case MAJOR:
2573               if (state == CompactionState.MINOR) {
2574                 return CompactionState.MAJOR_AND_MINOR;
2575               }
2576               state = CompactionState.MAJOR;
2577               break;
2578             case MINOR:
2579               if (state == CompactionState.MAJOR) {
2580                 return CompactionState.MAJOR_AND_MINOR;
2581               }
2582               state = CompactionState.MINOR;
2583               break;
2584             case NONE:
2585               default: // nothing, continue
2586             }
2587           } catch (NotServingRegionException e) {
2588             if (LOG.isDebugEnabled()) {
2589               LOG.debug("Trying to get compaction state of " +
2590                 pair.getFirst() + ": " +
2591                 StringUtils.stringifyException(e));
2592             }
2593           } catch (RemoteException e) {
2594             if (e.getMessage().indexOf(NotServingRegionException.class.getName()) >= 0) {
2595               if (LOG.isDebugEnabled()) {
2596                 LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": "
2597                     + StringUtils.stringifyException(e));
2598               }
2599             } else {
2600               throw e;
2601             }
2602           }
2603         }
2604       }
2605     } catch (ServiceException se) {
2606       throw ProtobufUtil.getRemoteException(se);
2607     } finally {
2608       cleanupCatalogTracker(ct);
2609     }
2610     return state;
2611   }
2612 
2613   /**
2614    * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be
2615    * taken. If the table is disabled, an offline snapshot is taken.
2616    * <p>
2617    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2618    * snapshot with the same name (even a different type or with different parameters) will fail with
2619    * a {@link SnapshotCreationException} indicating the duplicate naming.
2620    * <p>
2621    * Snapshot names follow the same naming constraints as tables in HBase. See
2622    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2623    * @param snapshotName name of the snapshot to be created
2624    * @param tableName name of the table for which snapshot is created
2625    * @throws IOException if a remote or network exception occurs
2626    * @throws SnapshotCreationException if snapshot creation failed
2627    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2628    */
2629   public void snapshot(final String snapshotName,
2630                        final TableName tableName) throws IOException,
2631       SnapshotCreationException, IllegalArgumentException {
2632     snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
2633   }
2634 
2635   public void snapshot(final String snapshotName,
2636                        final String tableName) throws IOException,
2637       SnapshotCreationException, IllegalArgumentException {
2638     snapshot(snapshotName, TableName.valueOf(tableName),
2639         SnapshotDescription.Type.FLUSH);
2640   }
2641 
2642   /**
2643    * Create snapshot for the given table of given flush type.
2644    * <p>
2645    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2646    * snapshot with the same name (even a different type or with different parameters) will fail with
2647    * a {@link SnapshotCreationException} indicating the duplicate naming.
2648    * <p>
2649    * Snapshot names follow the same naming constraints as tables in HBase.
2650    * @param snapshotName name of the snapshot to be created
2651    * @param tableName name of the table for which snapshot is created
2652    * @param flushType if the snapshot should be taken without flush memstore first
2653    * @throws IOException if a remote or network exception occurs
2654    * @throws SnapshotCreationException if snapshot creation failed
2655    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2656    */
2657    public void snapshot(final byte[] snapshotName, final byte[] tableName,
2658                        final SnapshotDescription.Type flushType) throws
2659       IOException, SnapshotCreationException, IllegalArgumentException {
2660       snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType);
2661   }
2662   /**
2663    public void snapshot(final String snapshotName,
2664     * Create a timestamp consistent snapshot for the given table.
2665                         final byte[] tableName) throws IOException,
2666     * <p>
2667     * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2668     * snapshot with the same name (even a different type or with different parameters) will fail with
2669     * a {@link SnapshotCreationException} indicating the duplicate naming.
2670     * <p>
2671     * Snapshot names follow the same naming constraints as tables in HBase.
2672     * @param snapshotName name of the snapshot to be created
2673     * @param tableName name of the table for which snapshot is created
2674     * @throws IOException if a remote or network exception occurs
2675     * @throws SnapshotCreationException if snapshot creation failed
2676     * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2677     */
2678   public void snapshot(final byte[] snapshotName,
2679                        final TableName tableName) throws IOException,
2680       SnapshotCreationException, IllegalArgumentException {
2681     snapshot(Bytes.toString(snapshotName), tableName, SnapshotDescription.Type.FLUSH);
2682   }
2683 
2684   public void snapshot(final byte[] snapshotName,
2685                        final byte[] tableName) throws IOException,
2686       SnapshotCreationException, IllegalArgumentException {
2687     snapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName),
2688         SnapshotDescription.Type.FLUSH);
2689   }
2690 
2691   /**
2692    * Create typed snapshot of the table.
2693    * <p>
2694    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2695    * snapshot with the same name (even a different type or with different parameters) will fail with
2696    * a {@link SnapshotCreationException} indicating the duplicate naming.
2697    * <p>
2698    * Snapshot names follow the same naming constraints as tables in HBase. See
2699    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2700    * <p>
2701    * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other
2702    *          snapshots stored on the cluster
2703    * @param tableName name of the table to snapshot
2704    * @param type type of snapshot to take
2705    * @throws IOException we fail to reach the master
2706    * @throws SnapshotCreationException if snapshot creation failed
2707    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2708    */
2709   public void snapshot(final String snapshotName,
2710                        final TableName tableName,
2711                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2712       IllegalArgumentException {
2713     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
2714     builder.setTable(tableName.getNameAsString());
2715     builder.setName(snapshotName);
2716     builder.setType(type);
2717     snapshot(builder.build());
2718   }
2719 
2720   public void snapshot(final String snapshotName,
2721                        final String tableName,
2722                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2723       IllegalArgumentException {
2724     snapshot(snapshotName, TableName.valueOf(tableName), type);
2725   }
2726 
2727   public void snapshot(final String snapshotName,
2728                        final byte[] tableName,
2729                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2730       IllegalArgumentException {
2731     snapshot(snapshotName, TableName.valueOf(tableName), type);
2732   }
2733 
2734   /**
2735    * Take a snapshot and wait for the server to complete that snapshot (blocking).
2736    * <p>
2737    * Only a single snapshot should be taken at a time for an instance of HBase, or results may be
2738    * undefined (you can tell multiple HBase clusters to snapshot at the same time, but only one at a
2739    * time for a single cluster).
2740    * <p>
2741    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2742    * snapshot with the same name (even a different type or with different parameters) will fail with
2743    * a {@link SnapshotCreationException} indicating the duplicate naming.
2744    * <p>
2745    * Snapshot names follow the same naming constraints as tables in HBase. See
2746    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2747    * <p>
2748    * You should probably use {@link #snapshot(String, String)} or {@link #snapshot(byte[], byte[])}
2749    * unless you are sure about the type of snapshot that you want to take.
2750    * @param snapshot snapshot to take
2751    * @throws IOException or we lose contact with the master.
2752    * @throws SnapshotCreationException if snapshot failed to be taken
2753    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2754    */
2755   public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException,
2756       IllegalArgumentException {
2757     // actually take the snapshot
2758     SnapshotResponse response = takeSnapshotAsync(snapshot);
2759     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot)
2760         .build();
2761     IsSnapshotDoneResponse done = null;
2762     long start = EnvironmentEdgeManager.currentTimeMillis();
2763     long max = response.getExpectedTimeout();
2764     long maxPauseTime = max / this.numRetries;
2765     int tries = 0;
2766     LOG.debug("Waiting a max of " + max + " ms for snapshot '" +
2767         ClientSnapshotDescriptionUtils.toString(snapshot) + "'' to complete. (max " +
2768         maxPauseTime + " ms per retry)");
2769     while (tries == 0
2770         || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done.getDone())) {
2771       try {
2772         // sleep a backoff <= pauseTime amount
2773         long sleep = getPauseTime(tries++);
2774         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2775         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
2776           "ms while waiting for snapshot completion.");
2777         Thread.sleep(sleep);
2778 
2779       } catch (InterruptedException e) {
2780         LOG.debug("Interrupted while waiting for snapshot " + snapshot + " to complete");
2781         Thread.currentThread().interrupt();
2782       }
2783       LOG.debug("Getting current status of snapshot from master...");
2784       done = executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2785         @Override
2786         public IsSnapshotDoneResponse call() throws ServiceException {
2787           return master.isSnapshotDone(null, request);
2788         }
2789       });
2790     };
2791     if (!done.getDone()) {
2792       throw new SnapshotCreationException("Snapshot '" + snapshot.getName()
2793           + "' wasn't completed in expectedTime:" + max + " ms", snapshot);
2794     }
2795   }
2796 
2797   /**
2798    * Take a snapshot without waiting for the server to complete that snapshot (asynchronous)
2799    * <p>
2800    * Only a single snapshot should be taken at a time, or results may be undefined.
2801    * @param snapshot snapshot to take
2802    * @return response from the server indicating the max time to wait for the snapshot
2803    * @throws IOException if the snapshot did not succeed or we lose contact with the master.
2804    * @throws SnapshotCreationException if snapshot creation failed
2805    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2806    */
2807   public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException,
2808       SnapshotCreationException {
2809     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
2810     final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot)
2811         .build();
2812     // run the snapshot on the master
2813     return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) {
2814       @Override
2815       public SnapshotResponse call() throws ServiceException {
2816         return master.snapshot(null, request);
2817       }
2818     });
2819   }
2820 
2821   /**
2822    * Check the current state of the passed snapshot.
2823    * <p>
2824    * There are three possible states:
2825    * <ol>
2826    * <li>running - returns <tt>false</tt></li>
2827    * <li>finished - returns <tt>true</tt></li>
2828    * <li>finished with error - throws the exception that caused the snapshot to fail</li>
2829    * </ol>
2830    * <p>
2831    * The cluster only knows about the most recent snapshot. Therefore, if another snapshot has been
2832    * run/started since the snapshot your are checking, you will recieve an
2833    * {@link UnknownSnapshotException}.
2834    * @param snapshot description of the snapshot to check
2835    * @return <tt>true</tt> if the snapshot is completed, <tt>false</tt> if the snapshot is still
2836    *         running
2837    * @throws IOException if we have a network issue
2838    * @throws HBaseSnapshotException if the snapshot failed
2839    * @throws UnknownSnapshotException if the requested snapshot is unknown
2840    */
2841   public boolean isSnapshotFinished(final SnapshotDescription snapshot)
2842       throws IOException, HBaseSnapshotException, UnknownSnapshotException {
2843 
2844     return executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2845       @Override
2846       public IsSnapshotDoneResponse call() throws ServiceException {
2847         return master.isSnapshotDone(null,
2848           IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build());
2849       }
2850     }).getDone();
2851   }
2852 
2853   /**
2854    * Restore the specified snapshot on the original table. (The table must be disabled)
2855    * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
2856    * is set to true, a snapshot of the current table is taken
2857    * before executing the restore operation.
2858    * In case of restore failure, the failsafe snapshot will be restored.
2859    * If the restore completes without problem the failsafe snapshot is deleted.
2860    *
2861    * @param snapshotName name of the snapshot to restore
2862    * @throws IOException if a remote or network exception occurs
2863    * @throws RestoreSnapshotException if snapshot failed to be restored
2864    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2865    */
2866   public void restoreSnapshot(final byte[] snapshotName)
2867       throws IOException, RestoreSnapshotException {
2868     restoreSnapshot(Bytes.toString(snapshotName));
2869   }
2870 
2871   /**
2872    * Restore the specified snapshot on the original table. (The table must be disabled)
2873    * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
2874    * is set to true, a snapshot of the current table is taken
2875    * before executing the restore operation.
2876    * In case of restore failure, the failsafe snapshot will be restored.
2877    * If the restore completes without problem the failsafe snapshot is deleted.
2878    *
2879    * @param snapshotName name of the snapshot to restore
2880    * @throws IOException if a remote or network exception occurs
2881    * @throws RestoreSnapshotException if snapshot failed to be restored
2882    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2883    */
2884   public void restoreSnapshot(final String snapshotName)
2885       throws IOException, RestoreSnapshotException {
2886     boolean takeFailSafeSnapshot =
2887       conf.getBoolean("hbase.snapshot.restore.take.failsafe.snapshot", false);
2888     restoreSnapshot(snapshotName, takeFailSafeSnapshot);
2889   }
2890 
2891   /**
2892    * Restore the specified snapshot on the original table. (The table must be disabled)
2893    * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
2894    * before executing the restore operation.
2895    * In case of restore failure, the failsafe snapshot will be restored.
2896    * If the restore completes without problem the failsafe snapshot is deleted.
2897    *
2898    * The failsafe snapshot name is configurable by using the property
2899    * "hbase.snapshot.restore.failsafe.name".
2900    *
2901    * @param snapshotName name of the snapshot to restore
2902    * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
2903    * @throws IOException if a remote or network exception occurs
2904    * @throws RestoreSnapshotException if snapshot failed to be restored
2905    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2906    */
2907   public void restoreSnapshot(final byte[] snapshotName, final boolean takeFailSafeSnapshot)
2908       throws IOException, RestoreSnapshotException {
2909     restoreSnapshot(Bytes.toString(snapshotName), takeFailSafeSnapshot);
2910   }
2911 
2912   /**
2913    * Restore the specified snapshot on the original table. (The table must be disabled)
2914    * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
2915    * before executing the restore operation.
2916    * In case of restore failure, the failsafe snapshot will be restored.
2917    * If the restore completes without problem the failsafe snapshot is deleted.
2918    *
2919    * The failsafe snapshot name is configurable by using the property
2920    * "hbase.snapshot.restore.failsafe.name".
2921    *
2922    * @param snapshotName name of the snapshot to restore
2923    * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
2924    * @throws IOException if a remote or network exception occurs
2925    * @throws RestoreSnapshotException if snapshot failed to be restored
2926    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2927    */
2928   public void restoreSnapshot(final String snapshotName, boolean takeFailSafeSnapshot)
2929       throws IOException, RestoreSnapshotException {
2930     TableName tableName = null;
2931     for (SnapshotDescription snapshotInfo: listSnapshots()) {
2932       if (snapshotInfo.getName().equals(snapshotName)) {
2933         tableName = TableName.valueOf(snapshotInfo.getTable());
2934         break;
2935       }
2936     }
2937 
2938     if (tableName == null) {
2939       throw new RestoreSnapshotException(
2940         "Unable to find the table name for snapshot=" + snapshotName);
2941     }
2942 
2943     // The table does not exists, switch to clone.
2944     if (!tableExists(tableName)) {
2945       try {
2946         cloneSnapshot(snapshotName, tableName);
2947       } catch (InterruptedException e) {
2948         throw new InterruptedIOException("Interrupted when restoring a nonexistent table: " +
2949           e.getMessage());
2950       }
2951       return;
2952     }
2953 
2954     // Check if the table is disabled
2955     if (!isTableDisabled(tableName)) {
2956       throw new TableNotDisabledException(tableName);
2957     }
2958 
2959     // Take a snapshot of the current state
2960     String failSafeSnapshotSnapshotName = null;
2961     if (takeFailSafeSnapshot) {
2962       failSafeSnapshotSnapshotName = conf.get("hbase.snapshot.restore.failsafe.name",
2963         "hbase-failsafe-{snapshot.name}-{restore.timestamp}");
2964       failSafeSnapshotSnapshotName = failSafeSnapshotSnapshotName
2965         .replace("{snapshot.name}", snapshotName)
2966         .replace("{table.name}", tableName.toString().replace(TableName.NAMESPACE_DELIM, '.'))
2967         .replace("{restore.timestamp}", String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
2968       LOG.info("Taking restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
2969       snapshot(failSafeSnapshotSnapshotName, tableName);
2970     }
2971 
2972     try {
2973       // Restore snapshot
2974       internalRestoreSnapshot(snapshotName, tableName);
2975     } catch (IOException e) {
2976       // Somthing went wrong during the restore...
2977       // if the pre-restore snapshot is available try to rollback
2978       if (takeFailSafeSnapshot) {
2979         try {
2980           internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName);
2981           String msg = "Restore snapshot=" + snapshotName +
2982             " failed. Rollback to snapshot=" + failSafeSnapshotSnapshotName + " succeeded.";
2983           LOG.error(msg, e);
2984           throw new RestoreSnapshotException(msg, e);
2985         } catch (IOException ex) {
2986           String msg = "Failed to restore and rollback to snapshot=" + failSafeSnapshotSnapshotName;
2987           LOG.error(msg, ex);
2988           throw new RestoreSnapshotException(msg, e);
2989         }
2990       } else {
2991         throw new RestoreSnapshotException("Failed to restore snapshot=" + snapshotName, e);
2992       }
2993     }
2994 
2995     // If the restore is succeeded, delete the pre-restore snapshot
2996     if (takeFailSafeSnapshot) {
2997       try {
2998         LOG.info("Deleting restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
2999         deleteSnapshot(failSafeSnapshotSnapshotName);
3000       } catch (IOException e) {
3001         LOG.error("Unable to remove the failsafe snapshot: " + failSafeSnapshotSnapshotName, e);
3002       }
3003     }
3004   }
3005 
3006   /**
3007    * Create a new table by cloning the snapshot content.
3008    *
3009    * @param snapshotName name of the snapshot to be cloned
3010    * @param tableName name of the table where the snapshot will be restored
3011    * @throws IOException if a remote or network exception occurs
3012    * @throws TableExistsException if table to be created already exists
3013    * @throws RestoreSnapshotException if snapshot failed to be cloned
3014    * @throws IllegalArgumentException if the specified table has not a valid name
3015    */
3016   public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName)
3017       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3018     cloneSnapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName));
3019   }
3020 
3021   /**
3022    * Create a new table by cloning the snapshot content.
3023    *
3024    * @param snapshotName name of the snapshot to be cloned
3025    * @param tableName name of the table where the snapshot will be restored
3026    * @throws IOException if a remote or network exception occurs
3027    * @throws TableExistsException if table to be created already exists
3028    * @throws RestoreSnapshotException if snapshot failed to be cloned
3029    * @throws IllegalArgumentException if the specified table has not a valid name
3030    */
3031   public void cloneSnapshot(final byte[] snapshotName, final TableName tableName)
3032       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3033     cloneSnapshot(Bytes.toString(snapshotName), tableName);
3034   }
3035 
3036 
3037 
3038   /**
3039    * Create a new table by cloning the snapshot content.
3040    *
3041    * @param snapshotName name of the snapshot to be cloned
3042    * @param tableName name of the table where the snapshot will be restored
3043    * @throws IOException if a remote or network exception occurs
3044    * @throws TableExistsException if table to be created already exists
3045    * @throws RestoreSnapshotException if snapshot failed to be cloned
3046    * @throws IllegalArgumentException if the specified table has not a valid name
3047    */
3048   public void cloneSnapshot(final String snapshotName, final String tableName)
3049       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3050     cloneSnapshot(snapshotName, TableName.valueOf(tableName));
3051   }
3052 
3053   /**
3054    * Create a new table by cloning the snapshot content.
3055    *
3056    * @param snapshotName name of the snapshot to be cloned
3057    * @param tableName name of the table where the snapshot will be restored
3058    * @throws IOException if a remote or network exception occurs
3059    * @throws TableExistsException if table to be created already exists
3060    * @throws RestoreSnapshotException if snapshot failed to be cloned
3061    * @throws IllegalArgumentException if the specified table has not a valid name
3062    */
3063   public void cloneSnapshot(final String snapshotName, final TableName tableName)
3064       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3065     if (tableExists(tableName)) {
3066       throw new TableExistsException(tableName);
3067     }
3068     internalRestoreSnapshot(snapshotName, tableName);
3069     waitUntilTableIsEnabled(tableName);
3070   }
3071 
3072   /**
3073    * Execute a distributed procedure on a cluster.
3074    *
3075    * @param signature A distributed procedure is uniquely identified
3076    * by its signature (default the root ZK node name of the procedure).
3077    * @param instance The instance name of the procedure. For some procedures, this parameter is
3078    * optional.
3079    * @param props Property/Value pairs of properties passing to the procedure
3080    */
3081   public void execProcedure(String signature, String instance,
3082       Map<String, String> props) throws IOException {
3083     ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
3084     builder.setSignature(signature).setInstance(instance);
3085     for (String key : props.keySet()) {
3086       NameStringPair pair = NameStringPair.newBuilder().setName(key)
3087           .setValue(props.get(key)).build();
3088       builder.addConfiguration(pair);
3089     }
3090 
3091     final ExecProcedureRequest request = ExecProcedureRequest.newBuilder()
3092         .setProcedure(builder.build()).build();
3093     // run the procedure on the master
3094     ExecProcedureResponse response = executeCallable(new MasterCallable<ExecProcedureResponse>(
3095         getConnection()) {
3096       @Override
3097       public ExecProcedureResponse call() throws ServiceException {
3098         return master.execProcedure(null, request);
3099       }
3100     });
3101 
3102     long start = EnvironmentEdgeManager.currentTimeMillis();
3103     long max = response.getExpectedTimeout();
3104     long maxPauseTime = max / this.numRetries;
3105     int tries = 0;
3106     LOG.debug("Waiting a max of " + max + " ms for procedure '" +
3107         signature + " : " + instance + "'' to complete. (max " + maxPauseTime + " ms per retry)");
3108     boolean done = false;
3109     while (tries == 0
3110         || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done)) {
3111       try {
3112         // sleep a backoff <= pauseTime amount
3113         long sleep = getPauseTime(tries++);
3114         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
3115         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
3116           "ms while waiting for procedure completion.");
3117         Thread.sleep(sleep);
3118 
3119       } catch (InterruptedException e) {
3120         LOG.debug("Interrupted while waiting for procedure " + signature + " to complete");
3121         Thread.currentThread().interrupt();
3122       }
3123       LOG.debug("Getting current status of procedure from master...");
3124       done = isProcedureFinished(signature, instance, props);
3125     }
3126     if (!done) {
3127       throw new IOException("Procedure '" + signature + " : " + instance
3128           + "' wasn't completed in expectedTime:" + max + " ms");
3129     }
3130   }
3131 
3132   /**
3133    * Check the current state of the specified procedure.
3134    * <p>
3135    * There are three possible states:
3136    * <ol>
3137    * <li>running - returns <tt>false</tt></li>
3138    * <li>finished - returns <tt>true</tt></li>
3139    * <li>finished with error - throws the exception that caused the procedure to fail</li>
3140    * </ol>
3141    * <p>
3142    *
3143    * @param signature The signature that uniquely identifies a procedure
3144    * @param instance The instance name of the procedure
3145    * @param props Property/Value pairs of properties passing to the procedure
3146    * @return true if the specified procedure is finished successfully, false if it is still running
3147    * @throws IOException if the specified procedure finished with error
3148    */
3149   public boolean isProcedureFinished(String signature, String instance, Map<String, String> props)
3150       throws IOException {
3151     final ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
3152     builder.setSignature(signature).setInstance(instance);
3153     for (String key : props.keySet()) {
3154       NameStringPair pair = NameStringPair.newBuilder().setName(key)
3155           .setValue(props.get(key)).build();
3156       builder.addConfiguration(pair);
3157     }
3158     final ProcedureDescription desc = builder.build();
3159     return executeCallable(
3160         new MasterCallable<IsProcedureDoneResponse>(getConnection()) {
3161           @Override
3162           public IsProcedureDoneResponse call() throws ServiceException {
3163             return master.isProcedureDone(null, IsProcedureDoneRequest
3164                 .newBuilder().setProcedure(desc).build());
3165           }
3166         }).getDone();
3167   }
3168 
3169   /**
3170    * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
3171    * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
3172    * create an HTable instance to this table before it is available.
3173    * @param snapshotName snapshot to restore
3174    * @param tableName table name to restore the snapshot on
3175    * @throws IOException if a remote or network exception occurs
3176    * @throws RestoreSnapshotException if snapshot failed to be restored
3177    * @throws IllegalArgumentException if the restore request is formatted incorrectly
3178    */
3179   private void internalRestoreSnapshot(final String snapshotName, final TableName
3180       tableName)
3181       throws IOException, RestoreSnapshotException {
3182     SnapshotDescription snapshot = SnapshotDescription.newBuilder()
3183         .setName(snapshotName).setTable(tableName.getNameAsString()).build();
3184 
3185     // actually restore the snapshot
3186     internalRestoreSnapshotAsync(snapshot);
3187 
3188     final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
3189         .setSnapshot(snapshot).build();
3190     IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
3191         .setDone(false).buildPartial();
3192     final long maxPauseTime = 5000;
3193     int tries = 0;
3194     while (!done.getDone()) {
3195       try {
3196         // sleep a backoff <= pauseTime amount
3197         long sleep = getPauseTime(tries++);
3198         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
3199         LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
3200         Thread.sleep(sleep);
3201       } catch (InterruptedException e) {
3202         LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete");
3203         Thread.currentThread().interrupt();
3204       }
3205       LOG.debug("Getting current status of snapshot restore from master...");
3206       done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
3207           getConnection()) {
3208         @Override
3209         public IsRestoreSnapshotDoneResponse call() throws ServiceException {
3210           return master.isRestoreSnapshotDone(null, request);
3211         }
3212       });
3213     }
3214     if (!done.getDone()) {
3215       throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
3216     }
3217   }
3218 
3219   /**
3220    * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
3221    * <p>
3222    * Only a single snapshot should be restored at a time, or results may be undefined.
3223    * @param snapshot snapshot to restore
3224    * @return response from the server indicating the max time to wait for the snapshot
3225    * @throws IOException if a remote or network exception occurs
3226    * @throws RestoreSnapshotException if snapshot failed to be restored
3227    * @throws IllegalArgumentException if the restore request is formatted incorrectly
3228    */
3229   private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
3230       throws IOException, RestoreSnapshotException {
3231     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
3232 
3233     final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
3234         .build();
3235 
3236     // run the snapshot restore on the master
3237     return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
3238       @Override
3239       public RestoreSnapshotResponse call() throws ServiceException {
3240         return master.restoreSnapshot(null, request);
3241       }
3242     });
3243   }
3244 
3245   /**
3246    * List completed snapshots.
3247    * @return a list of snapshot descriptors for completed snapshots
3248    * @throws IOException if a network error occurs
3249    */
3250   public List<SnapshotDescription> listSnapshots() throws IOException {
3251     return executeCallable(new MasterCallable<List<SnapshotDescription>>(getConnection()) {
3252       @Override
3253       public List<SnapshotDescription> call() throws ServiceException {
3254         return master.getCompletedSnapshots(null, GetCompletedSnapshotsRequest.newBuilder().build())
3255             .getSnapshotsList();
3256       }
3257     });
3258   }
3259 
3260   /**
3261    * List all the completed snapshots matching the given regular expression.
3262    *
3263    * @param regex The regular expression to match against
3264    * @return - returns a List of SnapshotDescription
3265    * @throws IOException if a remote or network exception occurs
3266    */
3267   public List<SnapshotDescription> listSnapshots(String regex) throws IOException {
3268     return listSnapshots(Pattern.compile(regex));
3269   }
3270 
3271   /**
3272    * List all the completed snapshots matching the given pattern.
3273    *
3274    * @param pattern The compiled regular expression to match against
3275    * @return - returns a List of SnapshotDescription
3276    * @throws IOException if a remote or network exception occurs
3277    */
3278   public List<SnapshotDescription> listSnapshots(Pattern pattern) throws IOException {
3279     List<SnapshotDescription> matched = new LinkedList<SnapshotDescription>();
3280     List<SnapshotDescription> snapshots = listSnapshots();
3281     for (SnapshotDescription snapshot : snapshots) {
3282       if (pattern.matcher(snapshot.getName()).matches()) {
3283         matched.add(snapshot);
3284       }
3285     }
3286     return matched;
3287   }
3288 
3289   /**
3290    * Delete an existing snapshot.
3291    * @param snapshotName name of the snapshot
3292    * @throws IOException if a remote or network exception occurs
3293    */
3294   public void deleteSnapshot(final byte[] snapshotName) throws IOException {
3295     deleteSnapshot(Bytes.toString(snapshotName));
3296   }
3297 
3298   /**
3299    * Delete an existing snapshot.
3300    * @param snapshotName name of the snapshot
3301    * @throws IOException if a remote or network exception occurs
3302    */
3303   public void deleteSnapshot(final String snapshotName) throws IOException {
3304     // make sure the snapshot is possibly valid
3305     TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(snapshotName));
3306     // do the delete
3307     executeCallable(new MasterCallable<Void>(getConnection()) {
3308       @Override
3309       public Void call() throws ServiceException {
3310         master.deleteSnapshot(null,
3311           DeleteSnapshotRequest.newBuilder().
3312             setSnapshot(SnapshotDescription.newBuilder().setName(snapshotName).build()).build());
3313         return null;
3314       }
3315     });
3316   }
3317 
3318   /**
3319    * Delete existing snapshots whose names match the pattern passed.
3320    * @param regex The regular expression to match against
3321    * @throws IOException if a remote or network exception occurs
3322    */
3323   public void deleteSnapshots(final String regex) throws IOException {
3324     deleteSnapshots(Pattern.compile(regex));
3325   }
3326 
3327   /**
3328    * Delete existing snapshots whose names match the pattern passed.
3329    * @param pattern pattern for names of the snapshot to match
3330    * @throws IOException if a remote or network exception occurs
3331    */
3332   public void deleteSnapshots(final Pattern pattern) throws IOException {
3333     List<SnapshotDescription> snapshots = listSnapshots(pattern);
3334     for (final SnapshotDescription snapshot : snapshots) {
3335       // do the delete
3336       executeCallable(new MasterCallable<Void>(getConnection()) {
3337         @Override
3338         public Void call() throws ServiceException {
3339           this.master.deleteSnapshot(null,
3340             DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot).build());
3341           return null;
3342         }
3343       });
3344     }
3345   }
3346 
3347   /**
3348    * Parent of {@link MasterCallable} and {@link MasterCallable}.
3349    * Has common methods.
3350    * @param <V>
3351    */
3352   abstract static class MasterCallable<V> implements RetryingCallable<V>, Closeable {
3353     protected HConnection connection;
3354     protected MasterKeepAliveConnection master;
3355 
3356     public MasterCallable(final HConnection connection) {
3357       this.connection = connection;
3358     }
3359 
3360     @Override
3361     public void prepare(boolean reload) throws IOException {
3362       this.master = this.connection.getKeepAliveMasterService();
3363     }
3364 
3365     @Override
3366     public void close() throws IOException {
3367       // The above prepare could fail but this would still be called though masterAdmin is null
3368       if (this.master != null) this.master.close();
3369     }
3370 
3371     @Override
3372     public void throwable(Throwable t, boolean retrying) {
3373     }
3374 
3375     @Override
3376     public String getExceptionMessageAdditionalDetail() {
3377       return "";
3378     }
3379 
3380     @Override
3381     public long sleep(long pause, int tries) {
3382       return ConnectionUtils.getPauseTime(pause, tries);
3383     }
3384   }
3385 
3386   private <V> V executeCallable(MasterCallable<V> callable) throws IOException {
3387     RpcRetryingCaller<V> caller = rpcCallerFactory.newCaller();
3388     try {
3389       return caller.callWithRetries(callable);
3390     } finally {
3391       callable.close();
3392     }
3393   }
3394 
3395   /**
3396    * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
3397    * connected to the active master.
3398    *
3399    * <p>
3400    * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
3401    * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
3402    * </p>
3403    *
3404    * <div style="background-color: #cccccc; padding: 2px">
3405    * <blockquote><pre>
3406    * CoprocessorRpcChannel channel = myAdmin.coprocessorService();
3407    * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
3408    * MyCallRequest request = MyCallRequest.newBuilder()
3409    *     ...
3410    *     .build();
3411    * MyCallResponse response = service.myCall(null, request);
3412    * </pre></blockquote></div>
3413    *
3414    * @return A MasterCoprocessorRpcChannel instance
3415    */
3416   public CoprocessorRpcChannel coprocessorService() {
3417     return new MasterCoprocessorRpcChannel(connection);
3418   }
3419 
3420   /**
3421    * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
3422    * connected to the passed region server.
3423    *
3424    * <p>
3425    * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
3426    * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
3427    * </p>
3428    *
3429    * <div style="background-color: #cccccc; padding: 2px">
3430    * <blockquote><pre>
3431    * CoprocessorRpcChannel channel = myAdmin.coprocessorService(serverName);
3432    * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
3433    * MyCallRequest request = MyCallRequest.newBuilder()
3434    *     ...
3435    *     .build();
3436    * MyCallResponse response = service.myCall(null, request);
3437    * </pre></blockquote></div>
3438    *
3439    * @param sn the server name to which the endpoint call is made
3440    * @return A RegionServerCoprocessorRpcChannel instance
3441    */
3442   public CoprocessorRpcChannel coprocessorService(ServerName sn) {
3443     return new RegionServerCoprocessorRpcChannel(connection, sn);
3444   }
3445 
3446   /**
3447    * Truncate a table. Synchronous operation.
3448    * @param tableName name of table to truncate
3449    * @param preserveSplits True if the splits should be preserved
3450    * @throws IOException if a remote or network exception occurs
3451    */
3452   public void truncateTable(final TableName tableName, final boolean preserveSplits)
3453       throws IOException {
3454     executeCallable(new MasterCallable<Void>(getConnection()) {
3455       @Override
3456       public Void call() throws ServiceException {
3457         LOG.info("Started truncate of " + tableName);
3458         TruncateTableRequest req = RequestConverter.buildTruncateTableRequest(
3459         tableName, preserveSplits);
3460         master.truncateTable(null, req);
3461         return null;
3462       }
3463     });
3464   }
3465 
3466 }