View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  import java.net.SocketTimeoutException;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.LinkedList;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.atomic.AtomicInteger;
32  import java.util.concurrent.atomic.AtomicReference;
33  import java.util.regex.Pattern;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.Abortable;
39  import org.apache.hadoop.hbase.ClusterStatus;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HBaseIOException;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HRegionLocation;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.MasterNotRunningException;
48  import org.apache.hadoop.hbase.NamespaceDescriptor;
49  import org.apache.hadoop.hbase.NotServingRegionException;
50  import org.apache.hadoop.hbase.RegionException;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.TableExistsException;
53  import org.apache.hadoop.hbase.TableName;
54  import org.apache.hadoop.hbase.TableNotDisabledException;
55  import org.apache.hadoop.hbase.TableNotEnabledException;
56  import org.apache.hadoop.hbase.TableNotFoundException;
57  import org.apache.hadoop.hbase.UnknownRegionException;
58  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
59  import org.apache.hadoop.hbase.catalog.CatalogTracker;
60  import org.apache.hadoop.hbase.catalog.MetaReader;
61  import org.apache.hadoop.hbase.classification.InterfaceAudience;
62  import org.apache.hadoop.hbase.classification.InterfaceStability;
63  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
64  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
65  import org.apache.hadoop.hbase.client.security.SecurityCapability;
66  import org.apache.hadoop.hbase.exceptions.DeserializationException;
67  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
68  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
69  import org.apache.hadoop.hbase.ipc.MasterCoprocessorRpcChannel;
70  import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel;
71  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
72  import org.apache.hadoop.hbase.protobuf.RequestConverter;
73  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
84  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
85  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
86  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
87  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
88  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
89  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
90  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
91  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceRequest;
92  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
93  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnRequest;
94  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest;
98  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest;
100 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest;
101 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureResponse;
102 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest;
103 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest;
105 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest;
106 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse;
107 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
111 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
113 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest;
117 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest;
118 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest;
119 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
120 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRequest;
121 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
122 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
124 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesRequest;
125 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
126 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
127 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest;
128 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
129 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
130 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableRequest;
131 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
132 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
133 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
134 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
135 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
136 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
137 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
138 import org.apache.hadoop.hbase.util.Addressing;
139 import org.apache.hadoop.hbase.util.Bytes;
140 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
141 import org.apache.hadoop.hbase.util.Pair;
142 import org.apache.hadoop.ipc.RemoteException;
143 import org.apache.hadoop.util.StringUtils;
144 import org.apache.zookeeper.KeeperException;
145 
146 import com.google.common.annotations.VisibleForTesting;
147 import com.google.protobuf.ByteString;
148 import com.google.protobuf.ServiceException;
149 
150 /**
151  * Provides an interface to manage HBase database table metadata + general
152  * administrative functions.  Use HBaseAdmin to create, drop, list, enable and
153  * disable tables. Use it also to add and drop table column families.
154  *
155  * <p>See {@link HTable} to add, update, and delete data from an individual table.
156  * <p>Currently HBaseAdmin instances are not expected to be long-lived.  For
157  * example, an HBaseAdmin instance will not ride over a Master restart.
158  */
159 @InterfaceAudience.Public
160 @InterfaceStability.Evolving
161 public class HBaseAdmin implements Abortable, Closeable {
162   private static final Log LOG = LogFactory.getLog(HBaseAdmin.class);
163 
164   // We use the implementation class rather then the interface because we
165   //  need the package protected functions to get the connection to master
166   private HConnection connection;
167 
168   private volatile Configuration conf;
169   private final long pause;
170   private final int numRetries;
171   // Some operations can take a long time such as disable of big table.
172   // numRetries is for 'normal' stuff... Multiply by this factor when
173   // want to wait a long time.
174   private final int retryLongerMultiplier;
175   private boolean aborted;
176   private boolean cleanupConnectionOnClose = false; // close the connection in close()
177   private boolean closed = false;
178 
179   private RpcRetryingCallerFactory rpcCallerFactory;
180 
181   /**
182    * Constructor.
183    * See {@link #HBaseAdmin(HConnection connection)}
184    *
185    * @param c Configuration object. Copied internally.
186    */
187   public HBaseAdmin(Configuration c)
188   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
189     // Will not leak connections, as the new implementation of the constructor
190     // does not throw exceptions anymore.
191     this(HConnectionManager.getConnection(new Configuration(c)));
192     this.cleanupConnectionOnClose = true;
193   }
194 
195  /**
196   * Constructor for externally managed HConnections.
197   * The connection to master will be created when required by admin functions.
198   *
199   * @param connection The HConnection instance to use
200   * @throws MasterNotRunningException, ZooKeeperConnectionException are not
201   *  thrown anymore but kept into the interface for backward api compatibility
202   */
203   public HBaseAdmin(HConnection connection)
204       throws MasterNotRunningException, ZooKeeperConnectionException {
205     this.conf = connection.getConfiguration();
206     this.connection = connection;
207 
208     this.pause = this.conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
209         HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
210     this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
211         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
212     this.retryLongerMultiplier = this.conf.getInt(
213         "hbase.client.retries.longer.multiplier", 10);
214     this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf,
215       connection.getStatisticsTracker());
216   }
217 
218   /**
219    * @return A new CatalogTracker instance; call {@link #cleanupCatalogTracker(CatalogTracker)}
220    * to cleanup the returned catalog tracker.
221    * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
222    * @throws IOException
223    * @see #cleanupCatalogTracker(CatalogTracker)
224    */
225   @VisibleForTesting
226   synchronized CatalogTracker getCatalogTracker()
227   throws ZooKeeperConnectionException, IOException {
228     boolean succeeded = false;
229     CatalogTracker ct = null;
230     try {
231       ct = new CatalogTracker(this.conf);
232       startCatalogTracker(ct);
233       succeeded = true;
234     } catch (InterruptedException e) {
235       // Let it out as an IOE for now until we redo all so tolerate IEs
236       throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
237     } finally {
238       // If we did not succeed but created a catalogtracker, clean it up. CT has a ZK instance
239       // in it and we'll leak if we don't do the 'stop'.
240       if (!succeeded && ct != null) {
241         try {
242           ct.stop();
243         } catch (RuntimeException re) {
244           LOG.error("Failed to clean up HBase's internal catalog tracker after a failed initialization. " +
245             "We may have leaked network connections to ZooKeeper; they won't be cleaned up until " +
246             "the JVM exits. If you see a large number of stale connections to ZooKeeper this is likely " +
247             "the cause. The following exception details will be needed for assistance from the " +
248             "HBase community.", re);
249         }
250         ct = null;
251       }
252     }
253     return ct;
254   }
255 
256   @VisibleForTesting
257   CatalogTracker startCatalogTracker(final CatalogTracker ct)
258   throws IOException, InterruptedException {
259     ct.start();
260     return ct;
261   }
262 
263   @VisibleForTesting
264   void cleanupCatalogTracker(final CatalogTracker ct) {
265     ct.stop();
266   }
267 
268   @Override
269   public void abort(String why, Throwable e) {
270     // Currently does nothing but throw the passed message and exception
271     this.aborted = true;
272     throw new RuntimeException(why, e);
273   }
274 
275   @Override
276   public boolean isAborted(){
277     return this.aborted;
278   }
279 
280   /** @return HConnection used by this object. */
281   public HConnection getConnection() {
282     return connection;
283   }
284 
285   /** @return - true if the master server is running. Throws an exception
286    *  otherwise.
287    * @throws ZooKeeperConnectionException
288    * @throws MasterNotRunningException
289    */
290   public boolean isMasterRunning()
291   throws MasterNotRunningException, ZooKeeperConnectionException {
292     return connection.isMasterRunning();
293   }
294 
295   /**
296    * @param tableName Table to check.
297    * @return True if table exists already.
298    * @throws IOException
299    */
300   public boolean tableExists(final TableName tableName)
301   throws IOException {
302     boolean b = false;
303     CatalogTracker ct = getCatalogTracker();
304     try {
305       b = MetaReader.tableExists(ct, tableName);
306     } finally {
307       cleanupCatalogTracker(ct);
308     }
309     return b;
310   }
311 
312   public boolean tableExists(final byte[] tableName)
313   throws IOException {
314     return tableExists(TableName.valueOf(tableName));
315   }
316 
317   public boolean tableExists(final String tableName)
318   throws IOException {
319     return tableExists(TableName.valueOf(tableName));
320   }
321 
322   /**
323    * List all the userspace tables.  In other words, scan the hbase:meta table.
324    *
325    * If we wanted this to be really fast, we could implement a special
326    * catalog table that just contains table names and their descriptors.
327    * Right now, it only exists as part of the hbase:meta table's region info.
328    *
329    * @return - returns an array of HTableDescriptors
330    * @throws IOException if a remote or network exception occurs
331    */
332   public HTableDescriptor[] listTables() throws IOException {
333     return this.connection.listTables();
334   }
335 
336   /**
337    * List all the userspace tables matching the given pattern.
338    *
339    * @param pattern The compiled regular expression to match against
340    * @return - returns an array of HTableDescriptors
341    * @throws IOException if a remote or network exception occurs
342    * @see #listTables()
343    */
344   public HTableDescriptor[] listTables(Pattern pattern) throws IOException {
345     List<HTableDescriptor> matched = new LinkedList<HTableDescriptor>();
346     HTableDescriptor[] tables = listTables();
347     for (HTableDescriptor table : tables) {
348       if (pattern.matcher(table.getTableName().getNameAsString()).matches()) {
349         matched.add(table);
350       }
351     }
352     return matched.toArray(new HTableDescriptor[matched.size()]);
353   }
354 
355   /**
356    * List all the userspace tables matching the given regular expression.
357    *
358    * @param regex The regular expression to match against
359    * @return - returns an array of HTableDescriptors
360    * @throws IOException if a remote or network exception occurs
361    * @see #listTables(java.util.regex.Pattern)
362    */
363   public HTableDescriptor[] listTables(String regex) throws IOException {
364     return listTables(Pattern.compile(regex));
365   }
366 
367   /**
368    * List all of the names of userspace tables.
369    * @return String[] table names
370    * @throws IOException if a remote or network exception occurs
371    */
372   @Deprecated
373   public String[] getTableNames() throws IOException {
374     return this.connection.getTableNames();
375   }
376 
377   /**
378    * List all of the names of userspace tables matching the given regular expression.
379    * @param pattern The regular expression to match against
380    * @return String[] table names
381    * @throws IOException if a remote or network exception occurs
382    */
383   @Deprecated
384   public String[] getTableNames(Pattern pattern) throws IOException {
385     List<String> matched = new ArrayList<String>();
386     for (String name: this.connection.getTableNames()) {
387       if (pattern.matcher(name).matches()) {
388         matched.add(name);
389       }
390     }
391     return matched.toArray(new String[matched.size()]);
392   }
393 
394   /**
395    * List all of the names of userspace tables matching the given regular expression.
396    * @param regex The regular expression to match against
397    * @return String[] table names
398    * @throws IOException if a remote or network exception occurs
399    */
400   @Deprecated
401   public String[] getTableNames(String regex) throws IOException {
402     return getTableNames(Pattern.compile(regex));
403   }
404 
405   /**
406    * List all of the names of userspace tables.
407    * @return TableName[] table names
408    * @throws IOException if a remote or network exception occurs
409    */
410   public TableName[] listTableNames() throws IOException {
411     return this.connection.listTableNames();
412   }
413 
414   /**
415    * Method for getting the tableDescriptor
416    * @param tableName as a byte []
417    * @return the tableDescriptor
418    * @throws TableNotFoundException
419    * @throws IOException if a remote or network exception occurs
420    */
421   public HTableDescriptor getTableDescriptor(final TableName tableName)
422   throws TableNotFoundException, IOException {
423     return this.connection.getHTableDescriptor(tableName);
424   }
425 
426   public HTableDescriptor getTableDescriptor(final byte[] tableName)
427   throws TableNotFoundException, IOException {
428     return getTableDescriptor(TableName.valueOf(tableName));
429   }
430 
431   private long getPauseTime(int tries) {
432     int triesCount = tries;
433     if (triesCount >= HConstants.RETRY_BACKOFF.length) {
434       triesCount = HConstants.RETRY_BACKOFF.length - 1;
435     }
436     return this.pause * HConstants.RETRY_BACKOFF[triesCount];
437   }
438 
439   /**
440    * Creates a new table.
441    * Synchronous operation.
442    *
443    * @param desc table descriptor for table
444    *
445    * @throws IllegalArgumentException if the table name is reserved
446    * @throws MasterNotRunningException if master is not running
447    * @throws TableExistsException if table already exists (If concurrent
448    * threads, the table may have been created between test-for-existence
449    * and attempt-at-creation).
450    * @throws IOException if a remote or network exception occurs
451    */
452   public void createTable(HTableDescriptor desc)
453   throws IOException {
454     createTable(desc, null);
455   }
456 
457   /**
458    * Creates a new table with the specified number of regions.  The start key
459    * specified will become the end key of the first region of the table, and
460    * the end key specified will become the start key of the last region of the
461    * table (the first region has a null start key and the last region has a
462    * null end key).
463    *
464    * BigInteger math will be used to divide the key range specified into
465    * enough segments to make the required number of total regions.
466    *
467    * Synchronous operation.
468    *
469    * @param desc table descriptor for table
470    * @param startKey beginning of key range
471    * @param endKey end of key range
472    * @param numRegions the total number of regions to create
473    *
474    * @throws IllegalArgumentException if the table name is reserved
475    * @throws MasterNotRunningException if master is not running
476    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
477    * threads, the table may have been created between test-for-existence
478    * and attempt-at-creation).
479    * @throws IOException
480    */
481   public void createTable(HTableDescriptor desc, byte [] startKey,
482       byte [] endKey, int numRegions)
483   throws IOException {
484     if(numRegions < 3) {
485       throw new IllegalArgumentException("Must create at least three regions");
486     } else if(Bytes.compareTo(startKey, endKey) >= 0) {
487       throw new IllegalArgumentException("Start key must be smaller than end key");
488     }
489     if (numRegions == 3) {
490       createTable(desc, new byte[][]{startKey, endKey});
491       return;
492     }
493     byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
494     if(splitKeys == null || splitKeys.length != numRegions - 1) {
495       throw new IllegalArgumentException("Unable to split key range into enough regions");
496     }
497     createTable(desc, splitKeys);
498   }
499 
500   /**
501    * Creates a new table with an initial set of empty regions defined by the
502    * specified split keys.  The total number of regions created will be the
503    * number of split keys plus one. Synchronous operation.
504    * Note : Avoid passing empty split key.
505    *
506    * @param desc table descriptor for table
507    * @param splitKeys array of split keys for the initial regions of the table
508    *
509    * @throws IllegalArgumentException if the table name is reserved, if the split keys
510    * are repeated and if the split key has empty byte array.
511    * @throws MasterNotRunningException if master is not running
512    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
513    * threads, the table may have been created between test-for-existence
514    * and attempt-at-creation).
515    * @throws IOException
516    */
517   public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
518   throws IOException {
519     try {
520       createTableAsync(desc, splitKeys);
521     } catch (SocketTimeoutException ste) {
522       LOG.warn("Creating " + desc.getTableName() + " took too long", ste);
523     }
524     int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
525     int prevRegCount = 0;
526     boolean doneWithMetaScan = false;
527     for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier;
528       ++tries) {
529       if (!doneWithMetaScan) {
530         // Wait for new table to come on-line
531         final AtomicInteger actualRegCount = new AtomicInteger(0);
532         MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
533           @Override
534           public boolean processRow(Result rowResult) throws IOException {
535             HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
536             if (info == null) {
537               LOG.warn("No serialized HRegionInfo in " + rowResult);
538               return true;
539             }
540             if (!info.getTable().equals(desc.getTableName())) {
541               return false;
542             }
543             ServerName serverName = HRegionInfo.getServerName(rowResult);
544             // Make sure that regions are assigned to server
545             if (!(info.isOffline() || info.isSplit()) && serverName != null
546                 && serverName.getHostAndPort() != null) {
547               actualRegCount.incrementAndGet();
548             }
549             return true;
550           }
551         };
552         MetaScanner.metaScan(conf, connection, visitor, desc.getTableName());
553         if (actualRegCount.get() < numRegs) {
554           if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
555             throw new RegionOfflineException("Only " + actualRegCount.get() +
556               " of " + numRegs + " regions are online; retries exhausted.");
557           }
558           try { // Sleep
559             Thread.sleep(getPauseTime(tries));
560           } catch (InterruptedException e) {
561             throw new InterruptedIOException("Interrupted when opening" +
562               " regions; " + actualRegCount.get() + " of " + numRegs +
563               " regions processed so far");
564           }
565           if (actualRegCount.get() > prevRegCount) { // Making progress
566             prevRegCount = actualRegCount.get();
567             tries = -1;
568           }
569         } else {
570           doneWithMetaScan = true;
571           tries = -1;
572         }
573       } else if (isTableEnabled(desc.getTableName())) {
574         return;
575       } else {
576         try { // Sleep
577           Thread.sleep(getPauseTime(tries));
578         } catch (InterruptedException e) {
579           throw new InterruptedIOException("Interrupted when waiting" +
580             " for table to be enabled; meta scan was done");
581         }
582       }
583     }
584     throw new TableNotEnabledException(
585       "Retries exhausted while still waiting for table: "
586       + desc.getTableName() + " to be enabled");
587   }
588 
589   /**
590    * Creates a new table but does not block and wait for it to come online.
591    * Asynchronous operation.  To check if the table exists, use
592    * {@link #isTableAvailable} -- it is not safe to create an HTable
593    * instance to this table before it is available.
594    * Note : Avoid passing empty split key.
595    * @param desc table descriptor for table
596    *
597    * @throws IllegalArgumentException Bad table name, if the split keys
598    * are repeated and if the split key has empty byte array.
599    * @throws MasterNotRunningException if master is not running
600    * @throws org.apache.hadoop.hbase.TableExistsException if table already exists (If concurrent
601    * threads, the table may have been created between test-for-existence
602    * and attempt-at-creation).
603    * @throws IOException
604    */
605   public void createTableAsync(
606     final HTableDescriptor desc, final byte [][] splitKeys)
607   throws IOException {
608     if(desc.getTableName() == null) {
609       throw new IllegalArgumentException("TableName cannot be null");
610     }
611     if(splitKeys != null && splitKeys.length > 0) {
612       Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
613       // Verify there are no duplicate split keys
614       byte [] lastKey = null;
615       for(byte [] splitKey : splitKeys) {
616         if (Bytes.compareTo(splitKey, HConstants.EMPTY_BYTE_ARRAY) == 0) {
617           throw new IllegalArgumentException(
618               "Empty split key must not be passed in the split keys.");
619         }
620         if(lastKey != null && Bytes.equals(splitKey, lastKey)) {
621           throw new IllegalArgumentException("All split keys must be unique, " +
622             "found duplicate: " + Bytes.toStringBinary(splitKey) +
623             ", " + Bytes.toStringBinary(lastKey));
624         }
625         lastKey = splitKey;
626       }
627     }
628 
629     executeCallable(new MasterCallable<Void>(getConnection()) {
630       @Override
631       public Void call() throws ServiceException {
632         CreateTableRequest request = RequestConverter.buildCreateTableRequest(desc, splitKeys);
633         master.createTable(null, request);
634         return null;
635       }
636     });
637   }
638 
639   public void deleteTable(final String tableName) throws IOException {
640     deleteTable(TableName.valueOf(tableName));
641   }
642 
643   public void deleteTable(final byte[] tableName) throws IOException {
644     deleteTable(TableName.valueOf(tableName));
645   }
646 
647   /**
648    * Deletes a table.
649    * Synchronous operation.
650    *
651    * @param tableName name of table to delete
652    * @throws IOException if a remote or network exception occurs
653    */
654   public void deleteTable(final TableName tableName) throws IOException {
655     boolean tableExists = true;
656 
657     executeCallable(new MasterCallable<Void>(getConnection()) {
658       @Override
659       public Void call() throws ServiceException {
660         DeleteTableRequest req = RequestConverter.buildDeleteTableRequest(tableName);
661         master.deleteTable(null,req);
662         return null;
663       }
664     });
665 
666     int failures = 0;
667     // Wait until all regions deleted
668     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
669       try {
670         final AtomicInteger count = new AtomicInteger(0);
671         MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
672           @Override
673           public boolean processRow(Result rowResult) throws IOException {
674             HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
675             if (info == null) {
676               return true;
677             }
678             if (!info.getTable().equals(tableName)) {
679               return false;
680             }
681             count.incrementAndGet();
682             return true;
683           }
684         };
685         MetaScanner.metaScan(conf, connection, visitor, tableName);
686         // let us wait until hbase:meta table is updated and
687         // HMaster removes the table from its HTableDescriptors
688         if (count.get() == 0) {
689           tableExists = false;
690           GetTableDescriptorsResponse htds;
691           MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
692           try {
693             GetTableDescriptorsRequest req =
694               RequestConverter.buildGetTableDescriptorsRequest(tableName);
695             htds = master.getTableDescriptors(null, req);
696           } catch (ServiceException se) {
697             throw ProtobufUtil.getRemoteException(se);
698           } finally {
699             master.close();
700           }
701           tableExists = !htds.getTableSchemaList().isEmpty();
702           if (!tableExists) {
703             break;
704           }
705         }
706       } catch (IOException ex) {
707         failures++;
708         if (failures == numRetries - 1) {           // no more tries left
709           if (ex instanceof RemoteException) {
710             ex = ((RemoteException) ex).unwrapRemoteException();
711           }
712           throw ex;
713         }
714       }
715       try {
716         Thread.sleep(getPauseTime(tries));
717       } catch (InterruptedException e) {
718         // continue
719       }
720     }
721 
722     if (tableExists) {
723       throw new IOException("Retries exhausted, it took too long to wait"+
724         " for the table " + tableName + " to be deleted.");
725     }
726     // Delete cached information to prevent clients from using old locations
727     this.connection.clearRegionCache(tableName);
728     LOG.info("Deleted " + tableName);
729   }
730 
731   /**
732    * Deletes tables matching the passed in pattern and wait on completion.
733    *
734    * Warning: Use this method carefully, there is no prompting and the effect is
735    * immediate. Consider using {@link #listTables(java.lang.String)} and
736    * {@link #deleteTable(byte[])}
737    *
738    * @param regex The regular expression to match table names against
739    * @return Table descriptors for tables that couldn't be deleted
740    * @throws IOException
741    * @see #deleteTables(java.util.regex.Pattern)
742    * @see #deleteTable(java.lang.String)
743    */
744   public HTableDescriptor[] deleteTables(String regex) throws IOException {
745     return deleteTables(Pattern.compile(regex));
746   }
747 
748   /**
749    * Delete tables matching the passed in pattern and wait on completion.
750    *
751    * Warning: Use this method carefully, there is no prompting and the effect is
752    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
753    * {@link #deleteTable(byte[])}
754    *
755    * @param pattern The pattern to match table names against
756    * @return Table descriptors for tables that couldn't be deleted
757    * @throws IOException
758    */
759   public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException {
760     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
761     for (HTableDescriptor table : listTables(pattern)) {
762       try {
763         deleteTable(table.getTableName());
764       } catch (IOException ex) {
765         LOG.info("Failed to delete table " + table.getTableName(), ex);
766         failed.add(table);
767       }
768     }
769     return failed.toArray(new HTableDescriptor[failed.size()]);
770   }
771 
772 
773   /**
774    * Enable a table.  May timeout.  Use {@link #enableTableAsync(byte[])}
775    * and {@link #isTableEnabled(byte[])} instead.
776    * The table has to be in disabled state for it to be enabled.
777    * @param tableName name of the table
778    * @throws IOException if a remote or network exception occurs
779    * There could be couple types of IOException
780    * TableNotFoundException means the table doesn't exist.
781    * TableNotDisabledException means the table isn't in disabled state.
782    * @see #isTableEnabled(byte[])
783    * @see #disableTable(byte[])
784    * @see #enableTableAsync(byte[])
785    */
786   public void enableTable(final TableName tableName)
787   throws IOException {
788     enableTableAsync(tableName);
789 
790     // Wait until all regions are enabled
791     waitUntilTableIsEnabled(tableName);
792 
793     LOG.info("Enabled table " + tableName);
794   }
795 
796   public void enableTable(final byte[] tableName)
797   throws IOException {
798     enableTable(TableName.valueOf(tableName));
799   }
800 
801   public void enableTable(final String tableName)
802   throws IOException {
803     enableTable(TableName.valueOf(tableName));
804   }
805 
806   /**
807    * Wait for the table to be enabled and available
808    * If enabling the table exceeds the retry period, an exception is thrown.
809    * @param tableName name of the table
810    * @throws IOException if a remote or network exception occurs or
811    *    table is not enabled after the retries period.
812    */
813   private void waitUntilTableIsEnabled(final TableName tableName) throws IOException {
814     boolean enabled = false;
815     long start = EnvironmentEdgeManager.currentTimeMillis();
816     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
817       try {
818         enabled = isTableEnabled(tableName);
819       } catch (TableNotFoundException tnfe) {
820         // wait for table to be created
821         enabled = false;
822       }
823       enabled = enabled && isTableAvailable(tableName);
824       if (enabled) {
825         break;
826       }
827       long sleep = getPauseTime(tries);
828       if (LOG.isDebugEnabled()) {
829         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
830           "enabled in " + tableName);
831       }
832       try {
833         Thread.sleep(sleep);
834       } catch (InterruptedException e) {
835         // Do this conversion rather than let it out because do not want to
836         // change the method signature.
837         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
838       }
839     }
840     if (!enabled) {
841       long msec = EnvironmentEdgeManager.currentTimeMillis() - start;
842       throw new IOException("Table '" + tableName +
843         "' not yet enabled, after " + msec + "ms.");
844     }
845   }
846 
847   /**
848    * Brings a table on-line (enables it).  Method returns immediately though
849    * enable of table may take some time to complete, especially if the table
850    * is large (All regions are opened as part of enabling process).  Check
851    * {@link #isTableEnabled(byte[])} to learn when table is fully online.  If
852    * table is taking too long to online, check server logs.
853    * @param tableName
854    * @throws IOException
855    * @since 0.90.0
856    */
857   public void enableTableAsync(final TableName tableName)
858   throws IOException {
859     TableName.isLegalFullyQualifiedTableName(tableName.getName());
860     executeCallable(new MasterCallable<Void>(getConnection()) {
861       @Override
862       public Void call() throws ServiceException {
863         LOG.info("Started enable of " + tableName);
864         EnableTableRequest req = RequestConverter.buildEnableTableRequest(tableName);
865         master.enableTable(null,req);
866         return null;
867       }
868     });
869   }
870 
871   public void enableTableAsync(final byte[] tableName)
872   throws IOException {
873     enableTable(TableName.valueOf(tableName));
874   }
875 
876   public void enableTableAsync(final String tableName)
877   throws IOException {
878     enableTableAsync(TableName.valueOf(tableName));
879   }
880 
881   /**
882    * Enable tables matching the passed in pattern and wait on completion.
883    *
884    * Warning: Use this method carefully, there is no prompting and the effect is
885    * immediate. Consider using {@link #listTables(java.lang.String)} and
886    * {@link #enableTable(byte[])}
887    *
888    * @param regex The regular expression to match table names against
889    * @throws IOException
890    * @see #enableTables(java.util.regex.Pattern)
891    * @see #enableTable(java.lang.String)
892    */
893   public HTableDescriptor[] enableTables(String regex) throws IOException {
894     return enableTables(Pattern.compile(regex));
895   }
896 
897   /**
898    * Enable tables matching the passed in pattern and wait on completion.
899    *
900    * Warning: Use this method carefully, there is no prompting and the effect is
901    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
902    * {@link #enableTable(byte[])}
903    *
904    * @param pattern The pattern to match table names against
905    * @throws IOException
906    */
907   public HTableDescriptor[] enableTables(Pattern pattern) throws IOException {
908     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
909     for (HTableDescriptor table : listTables(pattern)) {
910       if (isTableDisabled(table.getTableName())) {
911         try {
912           enableTable(table.getTableName());
913         } catch (IOException ex) {
914           LOG.info("Failed to enable table " + table.getTableName(), ex);
915           failed.add(table);
916         }
917       }
918     }
919     return failed.toArray(new HTableDescriptor[failed.size()]);
920   }
921 
922   /**
923    * Starts the disable of a table.  If it is being served, the master
924    * will tell the servers to stop serving it.  This method returns immediately.
925    * The disable of a table can take some time if the table is large (all
926    * regions are closed as part of table disable operation).
927    * Call {@link #isTableDisabled(byte[])} to check for when disable completes.
928    * If table is taking too long to online, check server logs.
929    * @param tableName name of table
930    * @throws IOException if a remote or network exception occurs
931    * @see #isTableDisabled(byte[])
932    * @see #isTableEnabled(byte[])
933    * @since 0.90.0
934    */
935   public void disableTableAsync(final TableName tableName) throws IOException {
936     TableName.isLegalFullyQualifiedTableName(tableName.getName());
937     executeCallable(new MasterCallable<Void>(getConnection()) {
938       @Override
939       public Void call() throws ServiceException {
940         LOG.info("Started disable of " + tableName);
941         DisableTableRequest req = RequestConverter.buildDisableTableRequest(tableName);
942         master.disableTable(null,req);
943         return null;
944       }
945     });
946   }
947 
948   public void disableTableAsync(final byte[] tableName) throws IOException {
949     disableTableAsync(TableName.valueOf(tableName));
950   }
951 
952   public void disableTableAsync(final String tableName) throws IOException {
953     disableTableAsync(TableName.valueOf(tableName));
954   }
955 
956   /**
957    * Disable table and wait on completion.  May timeout eventually.  Use
958    * {@link #disableTableAsync(byte[])} and {@link #isTableDisabled(String)}
959    * instead.
960    * The table has to be in enabled state for it to be disabled.
961    * @param tableName
962    * @throws IOException
963    * There could be couple types of IOException
964    * TableNotFoundException means the table doesn't exist.
965    * TableNotEnabledException means the table isn't in enabled state.
966    */
967   public void disableTable(final TableName tableName)
968   throws IOException {
969     disableTableAsync(tableName);
970     // Wait until table is disabled
971     boolean disabled = false;
972     for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) {
973       disabled = isTableDisabled(tableName);
974       if (disabled) {
975         break;
976       }
977       long sleep = getPauseTime(tries);
978       if (LOG.isDebugEnabled()) {
979         LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
980           "disabled in " + tableName);
981       }
982       try {
983         Thread.sleep(sleep);
984       } catch (InterruptedException e) {
985         // Do this conversion rather than let it out because do not want to
986         // change the method signature.
987         throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e);
988       }
989     }
990     if (!disabled) {
991       throw new RegionException("Retries exhausted, it took too long to wait"+
992         " for the table " + tableName + " to be disabled.");
993     }
994     LOG.info("Disabled " + tableName);
995   }
996 
997   public void disableTable(final byte[] tableName)
998   throws IOException {
999     disableTable(TableName.valueOf(tableName));
1000   }
1001 
1002   public void disableTable(final String tableName)
1003   throws IOException {
1004     disableTable(TableName.valueOf(tableName));
1005   }
1006 
1007   /**
1008    * Disable tables matching the passed in pattern and wait on completion.
1009    *
1010    * Warning: Use this method carefully, there is no prompting and the effect is
1011    * immediate. Consider using {@link #listTables(java.lang.String)} and
1012    * {@link #disableTable(byte[])}
1013    *
1014    * @param regex The regular expression to match table names against
1015    * @return Table descriptors for tables that couldn't be disabled
1016    * @throws IOException
1017    * @see #disableTables(java.util.regex.Pattern)
1018    * @see #disableTable(java.lang.String)
1019    */
1020   public HTableDescriptor[] disableTables(String regex) throws IOException {
1021     return disableTables(Pattern.compile(regex));
1022   }
1023 
1024   /**
1025    * Disable tables matching the passed in pattern and wait on completion.
1026    *
1027    * Warning: Use this method carefully, there is no prompting and the effect is
1028    * immediate. Consider using {@link #listTables(java.util.regex.Pattern) } and
1029    * {@link #disableTable(byte[])}
1030    *
1031    * @param pattern The pattern to match table names against
1032    * @return Table descriptors for tables that couldn't be disabled
1033    * @throws IOException
1034    */
1035   public HTableDescriptor[] disableTables(Pattern pattern) throws IOException {
1036     List<HTableDescriptor> failed = new LinkedList<HTableDescriptor>();
1037     for (HTableDescriptor table : listTables(pattern)) {
1038       if (isTableEnabled(table.getTableName())) {
1039         try {
1040           disableTable(table.getTableName());
1041         } catch (IOException ex) {
1042           LOG.info("Failed to disable table " + table.getTableName(), ex);
1043           failed.add(table);
1044         }
1045       }
1046     }
1047     return failed.toArray(new HTableDescriptor[failed.size()]);
1048   }
1049 
1050   /*
1051    * Checks whether table exists. If not, throws TableNotFoundException
1052    * @param tableName
1053    */
1054   private void checkTableExistence(TableName tableName) throws IOException {
1055     if (!tableExists(tableName)) {
1056       throw new TableNotFoundException(tableName);
1057     }
1058   }
1059 
1060   /**
1061    * @param tableName name of table to check
1062    * @return true if table is on-line
1063    * @throws IOException if a remote or network exception occurs
1064    */
1065   public boolean isTableEnabled(TableName tableName) throws IOException {
1066     checkTableExistence(tableName);
1067     return connection.isTableEnabled(tableName);
1068   }
1069 
1070   public boolean isTableEnabled(byte[] tableName) throws IOException {
1071     return isTableEnabled(TableName.valueOf(tableName));
1072   }
1073 
1074   public boolean isTableEnabled(String tableName) throws IOException {
1075     return isTableEnabled(TableName.valueOf(tableName));
1076   }
1077 
1078 
1079 
1080   /**
1081    * @param tableName name of table to check
1082    * @return true if table is off-line
1083    * @throws IOException if a remote or network exception occurs
1084    */
1085   public boolean isTableDisabled(TableName tableName) throws IOException {
1086     checkTableExistence(tableName);
1087     return connection.isTableDisabled(tableName);
1088   }
1089 
1090   public boolean isTableDisabled(byte[] tableName) throws IOException {
1091     return isTableDisabled(TableName.valueOf(tableName));
1092   }
1093 
1094   public boolean isTableDisabled(String tableName) throws IOException {
1095     return isTableDisabled(TableName.valueOf(tableName));
1096   }
1097 
1098   /**
1099    * @param tableName name of table to check
1100    * @return true if all regions of the table are available
1101    * @throws IOException if a remote or network exception occurs
1102    */
1103   public boolean isTableAvailable(TableName tableName) throws IOException {
1104     return connection.isTableAvailable(tableName);
1105   }
1106 
1107   public boolean isTableAvailable(byte[] tableName) throws IOException {
1108     return isTableAvailable(TableName.valueOf(tableName));
1109   }
1110 
1111   public boolean isTableAvailable(String tableName) throws IOException {
1112     return isTableAvailable(TableName.valueOf(tableName));
1113   }
1114 
1115   /**
1116    * Use this api to check if the table has been created with the specified number of
1117    * splitkeys which was used while creating the given table.
1118    * Note : If this api is used after a table's region gets splitted, the api may return
1119    * false.
1120    * @param tableName
1121    *          name of table to check
1122    * @param splitKeys
1123    *          keys to check if the table has been created with all split keys
1124    * @throws IOException
1125    *           if a remote or network excpetion occurs
1126    */
1127   public boolean isTableAvailable(TableName tableName,
1128                                   byte[][] splitKeys) throws IOException {
1129     return connection.isTableAvailable(tableName, splitKeys);
1130   }
1131 
1132   public boolean isTableAvailable(byte[] tableName,
1133                                   byte[][] splitKeys) throws IOException {
1134     return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1135   }
1136 
1137   public boolean isTableAvailable(String tableName,
1138                                   byte[][] splitKeys) throws IOException {
1139     return isTableAvailable(TableName.valueOf(tableName), splitKeys);
1140   }
1141 
1142   /**
1143    * Get the status of alter command - indicates how many regions have received
1144    * the updated schema Asynchronous operation.
1145    *
1146    * @param tableName TableName instance
1147    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1148    *         regions that are yet to be updated Pair.getSecond() is the total number
1149    *         of regions of the table
1150    * @throws IOException
1151    *           if a remote or network exception occurs
1152    */
1153   public Pair<Integer, Integer> getAlterStatus(final TableName tableName)
1154   throws IOException {
1155     return executeCallable(new MasterCallable<Pair<Integer, Integer>>(getConnection()) {
1156       @Override
1157       public Pair<Integer, Integer> call() throws ServiceException {
1158         GetSchemaAlterStatusRequest req = RequestConverter
1159             .buildGetSchemaAlterStatusRequest(tableName);
1160         GetSchemaAlterStatusResponse ret = master.getSchemaAlterStatus(null, req);
1161         Pair<Integer, Integer> pair = new Pair<Integer, Integer>(Integer.valueOf(ret
1162             .getYetToUpdateRegions()), Integer.valueOf(ret.getTotalRegions()));
1163         return pair;
1164       }
1165     });
1166   }
1167 
1168   /**
1169    * Get the status of alter command - indicates how many regions have received
1170    * the updated schema Asynchronous operation.
1171    *
1172    * @param tableName
1173    *          name of the table to get the status of
1174    * @return Pair indicating the number of regions updated Pair.getFirst() is the
1175    *         regions that are yet to be updated Pair.getSecond() is the total number
1176    *         of regions of the table
1177    * @throws IOException
1178    *           if a remote or network exception occurs
1179    */
1180   public Pair<Integer, Integer> getAlterStatus(final byte[] tableName)
1181    throws IOException {
1182     return getAlterStatus(TableName.valueOf(tableName));
1183   }
1184 
1185   /**
1186    * Add a column to an existing table.
1187    * Asynchronous operation.
1188    *
1189    * @param tableName name of the table to add column to
1190    * @param column column descriptor of column to be added
1191    * @throws IOException if a remote or network exception occurs
1192    */
1193   public void addColumn(final byte[] tableName, HColumnDescriptor column)
1194   throws IOException {
1195     addColumn(TableName.valueOf(tableName), column);
1196   }
1197 
1198 
1199   /**
1200    * Add a column to an existing table.
1201    * Asynchronous operation.
1202    *
1203    * @param tableName name of the table to add column to
1204    * @param column column descriptor of column to be added
1205    * @throws IOException if a remote or network exception occurs
1206    */
1207   public void addColumn(final String tableName, HColumnDescriptor column)
1208   throws IOException {
1209     addColumn(TableName.valueOf(tableName), column);
1210   }
1211 
1212   /**
1213    * Add a column to an existing table.
1214    * Asynchronous operation.
1215    *
1216    * @param tableName name of the table to add column to
1217    * @param column column descriptor of column to be added
1218    * @throws IOException if a remote or network exception occurs
1219    */
1220   public void addColumn(final TableName tableName, final HColumnDescriptor column)
1221   throws IOException {
1222     executeCallable(new MasterCallable<Void>(getConnection()) {
1223       @Override
1224       public Void call() throws ServiceException {
1225         AddColumnRequest req = RequestConverter.buildAddColumnRequest(tableName, column);
1226         master.addColumn(null,req);
1227         return null;
1228       }
1229     });
1230   }
1231 
1232   /**
1233    * Delete a column from a table.
1234    * Asynchronous operation.
1235    *
1236    * @param tableName name of table
1237    * @param columnName name of column to be deleted
1238    * @throws IOException if a remote or network exception occurs
1239    */
1240   public void deleteColumn(final byte[] tableName, final String columnName)
1241   throws IOException {
1242     deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
1243   }
1244 
1245   /**
1246    * Delete a column from a table.
1247    * Asynchronous operation.
1248    *
1249    * @param tableName name of table
1250    * @param columnName name of column to be deleted
1251    * @throws IOException if a remote or network exception occurs
1252    */
1253   public void deleteColumn(final String tableName, final String columnName)
1254   throws IOException {
1255     deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName));
1256   }
1257 
1258   /**
1259    * Delete a column from a table.
1260    * Asynchronous operation.
1261    *
1262    * @param tableName name of table
1263    * @param columnName name of column to be deleted
1264    * @throws IOException if a remote or network exception occurs
1265    */
1266   public void deleteColumn(final TableName tableName, final byte [] columnName)
1267   throws IOException {
1268     executeCallable(new MasterCallable<Void>(getConnection()) {
1269       @Override
1270       public Void call() throws ServiceException {
1271         DeleteColumnRequest req = RequestConverter.buildDeleteColumnRequest(tableName, columnName);
1272         master.deleteColumn(null,req);
1273         return null;
1274       }
1275     });
1276   }
1277 
1278   /**
1279    * Modify an existing column family on a table.
1280    * Asynchronous operation.
1281    *
1282    * @param tableName name of table
1283    * @param descriptor new column descriptor to use
1284    * @throws IOException if a remote or network exception occurs
1285    */
1286   public void modifyColumn(final String tableName, HColumnDescriptor descriptor)
1287   throws IOException {
1288     modifyColumn(TableName.valueOf(tableName), descriptor);
1289   }
1290 
1291   /**
1292    * Modify an existing column family on a table.
1293    * Asynchronous operation.
1294    *
1295    * @param tableName name of table
1296    * @param descriptor new column descriptor to use
1297    * @throws IOException if a remote or network exception occurs
1298    */
1299   public void modifyColumn(final byte[] tableName, HColumnDescriptor descriptor)
1300   throws IOException {
1301     modifyColumn(TableName.valueOf(tableName), descriptor);
1302   }
1303 
1304 
1305 
1306   /**
1307    * Modify an existing column family on a table.
1308    * Asynchronous operation.
1309    *
1310    * @param tableName name of table
1311    * @param descriptor new column descriptor to use
1312    * @throws IOException if a remote or network exception occurs
1313    */
1314   public void modifyColumn(final TableName tableName, final HColumnDescriptor descriptor)
1315   throws IOException {
1316     executeCallable(new MasterCallable<Void>(getConnection()) {
1317       @Override
1318       public Void call() throws ServiceException {
1319         ModifyColumnRequest req = RequestConverter.buildModifyColumnRequest(tableName, descriptor);
1320         master.modifyColumn(null,req);
1321         return null;
1322       }
1323     });
1324   }
1325 
1326   /**
1327    * Close a region. For expert-admins.  Runs close on the regionserver.  The
1328    * master will not be informed of the close.
1329    * @param regionname region name to close
1330    * @param serverName If supplied, we'll use this location rather than
1331    * the one currently in <code>hbase:meta</code>
1332    * @throws IOException if a remote or network exception occurs
1333    */
1334   public void closeRegion(final String regionname, final String serverName)
1335   throws IOException {
1336     closeRegion(Bytes.toBytes(regionname), serverName);
1337   }
1338 
1339   /**
1340    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1341    * master will not be informed of the close.
1342    * @param regionname region name to close
1343    * @param serverName The servername of the regionserver.  If passed null we
1344    * will use servername found in the hbase:meta table. A server name
1345    * is made of host, port and startcode.  Here is an example:
1346    * <code> host187.example.com,60020,1289493121758</code>
1347    * @throws IOException if a remote or network exception occurs
1348    */
1349   public void closeRegion(final byte [] regionname, final String serverName)
1350   throws IOException {
1351     CatalogTracker ct = getCatalogTracker();
1352     try {
1353       if (serverName != null) {
1354         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1355         if (pair == null || pair.getFirst() == null) {
1356           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1357         } else {
1358           closeRegion(ServerName.valueOf(serverName), pair.getFirst());
1359         }
1360       } else {
1361         Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, regionname);
1362         if (pair == null) {
1363           throw new UnknownRegionException(Bytes.toStringBinary(regionname));
1364         } else if (pair.getSecond() == null) {
1365           throw new NoServerForRegionException(Bytes.toStringBinary(regionname));
1366         } else {
1367           closeRegion(pair.getSecond(), pair.getFirst());
1368         }
1369       }
1370     } finally {
1371       cleanupCatalogTracker(ct);
1372     }
1373   }
1374 
1375   /**
1376    * For expert-admins. Runs close on the regionserver. Closes a region based on
1377    * the encoded region name. The region server name is mandatory. If the
1378    * servername is provided then based on the online regions in the specified
1379    * regionserver the specified region will be closed. The master will not be
1380    * informed of the close. Note that the regionname is the encoded regionname.
1381    *
1382    * @param encodedRegionName
1383    *          The encoded region name; i.e. the hash that makes up the region
1384    *          name suffix: e.g. if regionname is
1385    *          <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>
1386    *          , then the encoded region name is:
1387    *          <code>527db22f95c8a9e0116f0cc13c680396</code>.
1388    * @param serverName
1389    *          The servername of the regionserver. A server name is made of host,
1390    *          port and startcode. This is mandatory. Here is an example:
1391    *          <code> host187.example.com,60020,1289493121758</code>
1392    * @return true if the region was closed, false if not.
1393    * @throws IOException
1394    *           if a remote or network exception occurs
1395    */
1396   public boolean closeRegionWithEncodedRegionName(final String encodedRegionName,
1397       final String serverName) throws IOException {
1398     if (null == serverName || ("").equals(serverName.trim())) {
1399       throw new IllegalArgumentException(
1400           "The servername cannot be null or empty.");
1401     }
1402     ServerName sn = ServerName.valueOf(serverName);
1403     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1404     // Close the region without updating zk state.
1405     CloseRegionRequest request =
1406       RequestConverter.buildCloseRegionRequest(sn, encodedRegionName, false);
1407     try {
1408       CloseRegionResponse response = admin.closeRegion(null, request);
1409       boolean isRegionClosed = response.getClosed();
1410       if (false == isRegionClosed) {
1411         LOG.error("Not able to close the region " + encodedRegionName + ".");
1412       }
1413       return isRegionClosed;
1414     } catch (ServiceException se) {
1415       throw ProtobufUtil.getRemoteException(se);
1416     }
1417   }
1418 
1419   /**
1420    * Close a region.  For expert-admins  Runs close on the regionserver.  The
1421    * master will not be informed of the close.
1422    * @param sn
1423    * @param hri
1424    * @throws IOException
1425    */
1426   public void closeRegion(final ServerName sn, final HRegionInfo hri)
1427   throws IOException {
1428     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1429     // Close the region without updating zk state.
1430     ProtobufUtil.closeRegion(admin, sn, hri.getRegionName(), false);
1431   }
1432 
1433   /**
1434    * Get all the online regions on a region server.
1435    */
1436   public List<HRegionInfo> getOnlineRegions(
1437       final ServerName sn) throws IOException {
1438     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1439     return ProtobufUtil.getOnlineRegions(admin);
1440   }
1441 
1442   /**
1443    * Flush a table or an individual region.
1444    * Synchronous operation.
1445    *
1446    * @param tableNameOrRegionName table or region to flush
1447    * @throws IOException if a remote or network exception occurs
1448    * @throws InterruptedException
1449    */
1450   public void flush(final String tableNameOrRegionName)
1451   throws IOException, InterruptedException {
1452     flush(Bytes.toBytes(tableNameOrRegionName));
1453   }
1454 
1455   /**
1456    * Flush a table or an individual region.
1457    * Synchronous operation.
1458    *
1459    * @param tableNameOrRegionName table or region to flush
1460    * @throws IOException if a remote or network exception occurs
1461    * @throws InterruptedException
1462    */
1463   public void flush(final byte[] tableNameOrRegionName)
1464   throws IOException, InterruptedException {
1465     CatalogTracker ct = getCatalogTracker();
1466     try {
1467       Pair<HRegionInfo, ServerName> regionServerPair
1468         = getRegion(tableNameOrRegionName, ct);
1469       if (regionServerPair != null) {
1470         if (regionServerPair.getSecond() == null) {
1471           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1472         } else {
1473           flush(regionServerPair.getSecond(), regionServerPair.getFirst());
1474         }
1475       } else {
1476         final TableName tableName = checkTableExists(
1477             TableName.valueOf(tableNameOrRegionName), ct);
1478         List<Pair<HRegionInfo, ServerName>> pairs =
1479           MetaReader.getTableRegionsAndLocations(ct,
1480               tableName);
1481         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1482           if (pair.getFirst().isOffline()) continue;
1483           if (pair.getSecond() == null) continue;
1484           try {
1485             flush(pair.getSecond(), pair.getFirst());
1486           } catch (NotServingRegionException e) {
1487             if (LOG.isDebugEnabled()) {
1488               LOG.debug("Trying to flush " + pair.getFirst() + ": " +
1489                 StringUtils.stringifyException(e));
1490             }
1491           }
1492         }
1493       }
1494     } finally {
1495       cleanupCatalogTracker(ct);
1496     }
1497   }
1498 
1499   private void flush(final ServerName sn, final HRegionInfo hri)
1500   throws IOException {
1501     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1502     FlushRegionRequest request =
1503       RequestConverter.buildFlushRegionRequest(hri.getRegionName());
1504     try {
1505       admin.flushRegion(null, request);
1506     } catch (ServiceException se) {
1507       throw ProtobufUtil.getRemoteException(se);
1508     }
1509   }
1510 
1511   /**
1512    * Compact a table or an individual region.
1513    * Asynchronous operation.
1514    *
1515    * @param tableNameOrRegionName table or region to compact
1516    * @throws IOException if a remote or network exception occurs
1517    * @throws InterruptedException
1518    */
1519   public void compact(final String tableNameOrRegionName)
1520   throws IOException, InterruptedException {
1521     compact(Bytes.toBytes(tableNameOrRegionName));
1522   }
1523 
1524   /**
1525    * Compact a table or an individual region.
1526    * Asynchronous operation.
1527    *
1528    * @param tableNameOrRegionName table or region to compact
1529    * @throws IOException if a remote or network exception occurs
1530    * @throws InterruptedException
1531    */
1532   public void compact(final byte[] tableNameOrRegionName)
1533   throws IOException, InterruptedException {
1534     compact(tableNameOrRegionName, null, false);
1535   }
1536 
1537   /**
1538    * Compact a column family within a table or region.
1539    * Asynchronous operation.
1540    *
1541    * @param tableOrRegionName table or region to compact
1542    * @param columnFamily column family within a table or region
1543    * @throws IOException if a remote or network exception occurs
1544    * @throws InterruptedException
1545    */
1546   public void compact(String tableOrRegionName, String columnFamily)
1547     throws IOException,  InterruptedException {
1548     compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily));
1549   }
1550 
1551   /**
1552    * Compact a column family within a table or region.
1553    * Asynchronous operation.
1554    *
1555    * @param tableNameOrRegionName table or region to compact
1556    * @param columnFamily column family within a table or region
1557    * @throws IOException if a remote or network exception occurs
1558    * @throws InterruptedException
1559    */
1560   public void compact(final byte[] tableNameOrRegionName, final byte[] columnFamily)
1561   throws IOException, InterruptedException {
1562     compact(tableNameOrRegionName, columnFamily, false);
1563   }
1564 
1565   /**
1566    * Major compact a table or an individual region.
1567    * Asynchronous operation.
1568    *
1569    * @param tableNameOrRegionName table or region to major compact
1570    * @throws IOException if a remote or network exception occurs
1571    * @throws InterruptedException
1572    */
1573   public void majorCompact(final String tableNameOrRegionName)
1574   throws IOException, InterruptedException {
1575     majorCompact(Bytes.toBytes(tableNameOrRegionName));
1576   }
1577 
1578   /**
1579    * Major compact a table or an individual region.
1580    * Asynchronous operation.
1581    *
1582    * @param tableNameOrRegionName table or region to major compact
1583    * @throws IOException if a remote or network exception occurs
1584    * @throws InterruptedException
1585    */
1586   public void majorCompact(final byte[] tableNameOrRegionName)
1587   throws IOException, InterruptedException {
1588     compact(tableNameOrRegionName, null, true);
1589   }
1590 
1591   /**
1592    * Major compact a column family within a table or region.
1593    * Asynchronous operation.
1594    *
1595    * @param tableNameOrRegionName table or region to major compact
1596    * @param columnFamily column family within a table or region
1597    * @throws IOException if a remote or network exception occurs
1598    * @throws InterruptedException
1599    */
1600   public void majorCompact(final String tableNameOrRegionName,
1601     final String columnFamily) throws IOException, InterruptedException {
1602     majorCompact(Bytes.toBytes(tableNameOrRegionName),
1603       Bytes.toBytes(columnFamily));
1604   }
1605 
1606   /**
1607    * Major compact a column family within a table or region.
1608    * Asynchronous operation.
1609    *
1610    * @param tableNameOrRegionName table or region to major compact
1611    * @param columnFamily column family within a table or region
1612    * @throws IOException if a remote or network exception occurs
1613    * @throws InterruptedException
1614    */
1615   public void majorCompact(final byte[] tableNameOrRegionName,
1616     final byte[] columnFamily) throws IOException, InterruptedException {
1617     compact(tableNameOrRegionName, columnFamily, true);
1618   }
1619 
1620   /**
1621    * Compact a table or an individual region.
1622    * Asynchronous operation.
1623    *
1624    * @param tableNameOrRegionName table or region to compact
1625    * @param columnFamily column family within a table or region
1626    * @param major True if we are to do a major compaction.
1627    * @throws IOException if a remote or network exception occurs
1628    * @throws InterruptedException
1629    */
1630   private void compact(final byte[] tableNameOrRegionName,
1631     final byte[] columnFamily,final boolean major)
1632   throws IOException, InterruptedException {
1633     CatalogTracker ct = getCatalogTracker();
1634     try {
1635       Pair<HRegionInfo, ServerName> regionServerPair
1636         = getRegion(tableNameOrRegionName, ct);
1637       if (regionServerPair != null) {
1638         if (regionServerPair.getSecond() == null) {
1639           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1640         } else {
1641           compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
1642         }
1643       } else {
1644         final TableName tableName =
1645             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
1646         List<Pair<HRegionInfo, ServerName>> pairs =
1647           MetaReader.getTableRegionsAndLocations(ct,
1648               tableName);
1649         for (Pair<HRegionInfo, ServerName> pair: pairs) {
1650           if (pair.getFirst().isOffline()) continue;
1651           if (pair.getSecond() == null) continue;
1652           try {
1653             compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
1654           } catch (NotServingRegionException e) {
1655             if (LOG.isDebugEnabled()) {
1656               LOG.debug("Trying to" + (major ? " major" : "") + " compact " +
1657                 pair.getFirst() + ": " +
1658                 StringUtils.stringifyException(e));
1659             }
1660           }
1661         }
1662       }
1663     } finally {
1664       cleanupCatalogTracker(ct);
1665     }
1666   }
1667 
1668   private void compact(final ServerName sn, final HRegionInfo hri,
1669       final boolean major, final byte [] family)
1670   throws IOException {
1671     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
1672     CompactRegionRequest request =
1673       RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major, family);
1674     try {
1675       admin.compactRegion(null, request);
1676     } catch (ServiceException se) {
1677       throw ProtobufUtil.getRemoteException(se);
1678     }
1679   }
1680 
1681   /**
1682    * Compact all regions on the region server
1683    * @param regionserver the region server name
1684    * @param major if it's major compaction
1685    * @throws IOException
1686    * @throws InterruptedException
1687    */
1688   public void compactRegionServer(final ServerName sn, boolean major)
1689       throws IOException, InterruptedException {
1690     for (HRegionInfo region : getOnlineRegions(sn)) {
1691       compact(sn, region, major, null);
1692     }
1693   }
1694 
1695   /**
1696    * Move the region <code>r</code> to <code>dest</code>.
1697    * @param encodedRegionName The encoded region name; i.e. the hash that makes
1698    * up the region name suffix: e.g. if regionname is
1699    * <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>,
1700    * then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>.
1701    * @param destServerName The servername of the destination regionserver.  If
1702    * passed the empty byte array we'll assign to a random server.  A server name
1703    * is made of host, port and startcode.  Here is an example:
1704    * <code> host187.example.com,60020,1289493121758</code>
1705    * @throws UnknownRegionException Thrown if we can't find a region named
1706    * <code>encodedRegionName</code>
1707    * @throws ZooKeeperConnectionException
1708    * @throws MasterNotRunningException
1709    */
1710   public void move(final byte [] encodedRegionName, final byte [] destServerName)
1711   throws HBaseIOException, MasterNotRunningException, ZooKeeperConnectionException {
1712     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1713     try {
1714       MoveRegionRequest request =
1715         RequestConverter.buildMoveRegionRequest(encodedRegionName, destServerName);
1716       stub.moveRegion(null,request);
1717     } catch (ServiceException se) {
1718       IOException ioe = ProtobufUtil.getRemoteException(se);
1719       if (ioe instanceof HBaseIOException) {
1720         throw (HBaseIOException)ioe;
1721       }
1722       LOG.error("Unexpected exception: " + se + " from calling HMaster.moveRegion");
1723     } catch (DeserializationException de) {
1724       LOG.error("Could not parse destination server name: " + de);
1725     } finally {
1726       stub.close();
1727     }
1728   }
1729 
1730   /**
1731    * @param regionName
1732    *          Region name to assign.
1733    * @throws MasterNotRunningException
1734    * @throws ZooKeeperConnectionException
1735    * @throws IOException
1736    */
1737   public void assign(final byte[] regionName) throws MasterNotRunningException,
1738       ZooKeeperConnectionException, IOException {
1739     final byte[] toBeAssigned = getRegionName(regionName);
1740     executeCallable(new MasterCallable<Void>(getConnection()) {
1741       @Override
1742       public Void call() throws ServiceException {
1743         AssignRegionRequest request =
1744           RequestConverter.buildAssignRegionRequest(toBeAssigned);
1745         master.assignRegion(null,request);
1746         return null;
1747       }
1748     });
1749   }
1750 
1751   /**
1752    * Unassign a region from current hosting regionserver.  Region will then be
1753    * assigned to a regionserver chosen at random.  Region could be reassigned
1754    * back to the same server.  Use {@link #move(byte[], byte[])} if you want
1755    * to control the region movement.
1756    * @param regionName Region to unassign. Will clear any existing RegionPlan
1757    * if one found.
1758    * @param force If true, force unassign (Will remove region from
1759    * regions-in-transition too if present. If results in double assignment
1760    * use hbck -fix to resolve. To be used by experts).
1761    * @throws MasterNotRunningException
1762    * @throws ZooKeeperConnectionException
1763    * @throws IOException
1764    */
1765   public void unassign(final byte [] regionName, final boolean force)
1766   throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
1767     final byte[] toBeUnassigned = getRegionName(regionName);
1768     executeCallable(new MasterCallable<Void>(getConnection()) {
1769       @Override
1770       public Void call() throws ServiceException {
1771         UnassignRegionRequest request =
1772           RequestConverter.buildUnassignRegionRequest(toBeUnassigned, force);
1773         master.unassignRegion(null,request);
1774         return null;
1775       }
1776     });
1777   }
1778 
1779   /**
1780    * Offline specified region from master's in-memory state. It will not attempt to reassign the
1781    * region as in unassign. This API can be used when a region not served by any region server and
1782    * still online as per Master's in memory state. If this API is incorrectly used on active region
1783    * then master will loose track of that region.
1784    *
1785    * This is a special method that should be used by experts or hbck.
1786    *
1787    * @param regionName
1788    *          Region to offline.
1789    * @throws IOException
1790    */
1791   public void offline(final byte [] regionName)
1792   throws IOException {
1793     MasterKeepAliveConnection master = connection.getKeepAliveMasterService();
1794     try {
1795       master.offlineRegion(null,RequestConverter.buildOfflineRegionRequest(regionName));
1796     } catch (ServiceException se) {
1797       throw ProtobufUtil.getRemoteException(se);
1798     } finally {
1799       master.close();
1800     }
1801   }
1802 
1803   /**
1804    * Turn the load balancer on or off.
1805    * @param on If true, enable balancer. If false, disable balancer.
1806    * @param synchronous If true, it waits until current balance() call, if outstanding, to return.
1807    * @return Previous balancer value
1808    */
1809   public boolean setBalancerRunning(final boolean on, final boolean synchronous)
1810   throws MasterNotRunningException, ZooKeeperConnectionException {
1811     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1812     try {
1813       SetBalancerRunningRequest req =
1814         RequestConverter.buildSetBalancerRunningRequest(on, synchronous);
1815       return stub.setBalancerRunning(null, req).getPrevBalanceValue();
1816     } catch (ServiceException se) {
1817       IOException ioe = ProtobufUtil.getRemoteException(se);
1818       if (ioe instanceof MasterNotRunningException) {
1819         throw (MasterNotRunningException)ioe;
1820       }
1821       if (ioe instanceof ZooKeeperConnectionException) {
1822         throw (ZooKeeperConnectionException)ioe;
1823       }
1824 
1825       // Throwing MasterNotRunningException even though not really valid in order to not
1826       // break interface by adding additional exception type.
1827       throw new MasterNotRunningException("Unexpected exception when calling balanceSwitch",se);
1828     } finally {
1829       stub.close();
1830     }
1831   }
1832 
1833   /**
1834    * Invoke the balancer.  Will run the balancer and if regions to move, it will
1835    * go ahead and do the reassignments.  Can NOT run for various reasons.  Check
1836    * logs.
1837    * @return True if balancer ran, false otherwise.
1838    */
1839   public boolean balancer()
1840   throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException {
1841     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1842     try {
1843       return stub.balance(null,RequestConverter.buildBalanceRequest()).getBalancerRan();
1844     } finally {
1845       stub.close();
1846     }
1847   }
1848 
1849   /**
1850    * Query the state of the balancer from the Master. It's not a guarantee that the balancer is
1851    * actually running this very moment, but that it will run.
1852    *
1853    * @return True if the balancer is enabled, false otherwise.
1854    */
1855   public boolean isBalancerEnabled()
1856       throws ServiceException, MasterNotRunningException {
1857     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1858     try {
1859       return stub.isBalancerEnabled(null, RequestConverter.buildIsBalancerEnabledRequest())
1860         .getEnabled();
1861     } finally {
1862       stub.close();
1863     }
1864   }
1865 
1866   /**
1867    * Enable/Disable the catalog janitor
1868    * @param enable if true enables the catalog janitor
1869    * @return the previous state
1870    * @throws ServiceException
1871    * @throws MasterNotRunningException
1872    */
1873   public boolean enableCatalogJanitor(boolean enable)
1874       throws ServiceException, MasterNotRunningException {
1875     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1876     try {
1877       return stub.enableCatalogJanitor(null,
1878           RequestConverter.buildEnableCatalogJanitorRequest(enable)).getPrevValue();
1879     } finally {
1880       stub.close();
1881     }
1882   }
1883 
1884   /**
1885    * Ask for a scan of the catalog table
1886    * @return the number of entries cleaned
1887    * @throws ServiceException
1888    * @throws MasterNotRunningException
1889    */
1890   public int runCatalogScan() throws ServiceException, MasterNotRunningException {
1891     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1892     try {
1893       return stub.runCatalogScan(null,
1894           RequestConverter.buildCatalogScanRequest()).getScanResult();
1895     } finally {
1896       stub.close();
1897     }
1898   }
1899 
1900   /**
1901    * Query on the catalog janitor state (Enabled/Disabled?)
1902    * @throws ServiceException
1903    * @throws org.apache.hadoop.hbase.MasterNotRunningException
1904    */
1905   public boolean isCatalogJanitorEnabled() throws ServiceException, MasterNotRunningException {
1906     MasterKeepAliveConnection stub = connection.getKeepAliveMasterService();
1907     try {
1908       return stub.isCatalogJanitorEnabled(null,
1909           RequestConverter.buildIsCatalogJanitorEnabledRequest()).getValue();
1910     } finally {
1911       stub.close();
1912     }
1913   }
1914 
1915   /**
1916    * Merge two regions. Asynchronous operation.
1917    * @param encodedNameOfRegionA encoded name of region a
1918    * @param encodedNameOfRegionB encoded name of region b
1919    * @param forcible true if do a compulsory merge, otherwise we will only merge
1920    *          two adjacent regions
1921    * @throws IOException
1922    */
1923   public void mergeRegions(final byte[] encodedNameOfRegionA,
1924       final byte[] encodedNameOfRegionB, final boolean forcible)
1925       throws IOException {
1926     MasterKeepAliveConnection master = connection
1927         .getKeepAliveMasterService();
1928     try {
1929       DispatchMergingRegionsRequest request = RequestConverter
1930           .buildDispatchMergingRegionsRequest(encodedNameOfRegionA,
1931               encodedNameOfRegionB, forcible);
1932       master.dispatchMergingRegions(null, request);
1933     } catch (ServiceException se) {
1934       IOException ioe = ProtobufUtil.getRemoteException(se);
1935       if (ioe instanceof UnknownRegionException) {
1936         throw (UnknownRegionException) ioe;
1937       }
1938       if (ioe instanceof MergeRegionException) {
1939         throw (MergeRegionException) ioe;
1940       }
1941       LOG.error("Unexpected exception: " + se
1942           + " from calling HMaster.dispatchMergingRegions");
1943     } catch (DeserializationException de) {
1944       LOG.error("Could not parse destination server name: " + de);
1945     } finally {
1946       master.close();
1947     }
1948   }
1949 
1950   /**
1951    * Split a table or an individual region.
1952    * Asynchronous operation.
1953    *
1954    * @param tableNameOrRegionName table or region to split
1955    * @throws IOException if a remote or network exception occurs
1956    * @throws InterruptedException
1957    */
1958   public void split(final String tableNameOrRegionName)
1959   throws IOException, InterruptedException {
1960     split(Bytes.toBytes(tableNameOrRegionName));
1961   }
1962 
1963   /**
1964    * Split a table or an individual region.  Implicitly finds an optimal split
1965    * point.  Asynchronous operation.
1966    *
1967    * @param tableNameOrRegionName table to region to split
1968    * @throws IOException if a remote or network exception occurs
1969    * @throws InterruptedException
1970    */
1971   public void split(final byte[] tableNameOrRegionName)
1972   throws IOException, InterruptedException {
1973     split(tableNameOrRegionName, null);
1974   }
1975 
1976   public void split(final String tableNameOrRegionName,
1977     final String splitPoint) throws IOException, InterruptedException {
1978     split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint));
1979   }
1980 
1981   /**
1982    * Split a table or an individual region.
1983    * Asynchronous operation.
1984    *
1985    * @param tableNameOrRegionName table to region to split
1986    * @param splitPoint the explicit position to split on
1987    * @throws IOException if a remote or network exception occurs
1988    * @throws InterruptedException interrupt exception occurred
1989    */
1990   public void split(final byte[] tableNameOrRegionName,
1991       final byte [] splitPoint) throws IOException, InterruptedException {
1992     CatalogTracker ct = getCatalogTracker();
1993     try {
1994       Pair<HRegionInfo, ServerName> regionServerPair
1995         = getRegion(tableNameOrRegionName, ct);
1996       if (regionServerPair != null) {
1997         if (regionServerPair.getSecond() == null) {
1998             throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
1999         } else {
2000           split(regionServerPair.getSecond(), regionServerPair.getFirst(), splitPoint);
2001         }
2002       } else {
2003         final TableName tableName =
2004             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
2005         List<Pair<HRegionInfo, ServerName>> pairs =
2006           MetaReader.getTableRegionsAndLocations(ct,
2007               tableName);
2008         for (Pair<HRegionInfo, ServerName> pair: pairs) {
2009           // May not be a server for a particular row
2010           if (pair.getSecond() == null) continue;
2011           HRegionInfo r = pair.getFirst();
2012           // check for parents
2013           if (r.isSplitParent()) continue;
2014           // if a split point given, only split that particular region
2015           if (splitPoint != null && !r.containsRow(splitPoint)) continue;
2016           // call out to region server to do split now
2017           split(pair.getSecond(), pair.getFirst(), splitPoint);
2018         }
2019       }
2020     } finally {
2021       cleanupCatalogTracker(ct);
2022     }
2023   }
2024 
2025   private void split(final ServerName sn, final HRegionInfo hri,
2026       byte[] splitPoint) throws IOException {
2027     if (hri.getStartKey() != null && splitPoint != null &&
2028          Bytes.compareTo(hri.getStartKey(), splitPoint) == 0) {
2029        throw new IOException("should not give a splitkey which equals to startkey!");
2030     }
2031     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2032     ProtobufUtil.split(admin, hri, splitPoint);
2033   }
2034 
2035   /**
2036    * Modify an existing table, more IRB friendly version.
2037    * Asynchronous operation.  This means that it may be a while before your
2038    * schema change is updated across all of the table.
2039    *
2040    * @param tableName name of table.
2041    * @param htd modified description of the table
2042    * @throws IOException if a remote or network exception occurs
2043    */
2044   public void modifyTable(final TableName tableName, final HTableDescriptor htd)
2045   throws IOException {
2046     if (!tableName.equals(htd.getTableName())) {
2047       throw new IllegalArgumentException("the specified table name '" + tableName +
2048         "' doesn't match with the HTD one: " + htd.getTableName());
2049     }
2050 
2051     executeCallable(new MasterCallable<Void>(getConnection()) {
2052       @Override
2053       public Void call() throws ServiceException {
2054         ModifyTableRequest request = RequestConverter.buildModifyTableRequest(tableName, htd);
2055         master.modifyTable(null, request);
2056         return null;
2057       }
2058     });
2059   }
2060 
2061   public void modifyTable(final byte[] tableName, final HTableDescriptor htd)
2062   throws IOException {
2063     modifyTable(TableName.valueOf(tableName), htd);
2064   }
2065 
2066   public void modifyTable(final String tableName, final HTableDescriptor htd)
2067   throws IOException {
2068     modifyTable(TableName.valueOf(tableName), htd);
2069   }
2070 
2071   /**
2072    * @param tableNameOrRegionName Name of a table or name of a region.
2073    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
2074    * @return a pair of HRegionInfo and ServerName if <code>tableNameOrRegionName</code> is
2075    *  a verified region name (we call {@link  MetaReader#getRegion( CatalogTracker, byte[])}
2076    *  else null.
2077    * Throw an exception if <code>tableNameOrRegionName</code> is null.
2078    * @throws IOException
2079    */
2080   Pair<HRegionInfo, ServerName> getRegion(final byte[] tableNameOrRegionName,
2081       final CatalogTracker ct) throws IOException {
2082     if (tableNameOrRegionName == null) {
2083       throw new IllegalArgumentException("Pass a table name or region name");
2084     }
2085     Pair<HRegionInfo, ServerName> pair = MetaReader.getRegion(ct, tableNameOrRegionName);
2086     if (pair == null) {
2087       final AtomicReference<Pair<HRegionInfo, ServerName>> result =
2088         new AtomicReference<Pair<HRegionInfo, ServerName>>(null);
2089       final String encodedName = Bytes.toString(tableNameOrRegionName);
2090       MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2091         @Override
2092         public boolean processRow(Result data) throws IOException {
2093           HRegionInfo info = HRegionInfo.getHRegionInfo(data);
2094           if (info == null) {
2095             LOG.warn("No serialized HRegionInfo in " + data);
2096             return true;
2097           }
2098           if (!encodedName.equals(info.getEncodedName())) return true;
2099           ServerName sn = HRegionInfo.getServerName(data);
2100           result.set(new Pair<HRegionInfo, ServerName>(info, sn));
2101           return false; // found the region, stop
2102         }
2103       };
2104 
2105       MetaScanner.metaScan(conf, connection, visitor, null);
2106       pair = result.get();
2107     }
2108     return pair;
2109   }
2110 
2111   /**
2112    * If the input is a region name, it is returned as is. If it's an
2113    * encoded region name, the corresponding region is found from meta
2114    * and its region name is returned. If we can't find any region in
2115    * meta matching the input as either region name or encoded region
2116    * name, the input is returned as is. We don't throw unknown
2117    * region exception.
2118    */
2119   private byte[] getRegionName(
2120       final byte[] regionNameOrEncodedRegionName) throws IOException {
2121     if (Bytes.equals(regionNameOrEncodedRegionName,
2122         HRegionInfo.FIRST_META_REGIONINFO.getRegionName())
2123           || Bytes.equals(regionNameOrEncodedRegionName,
2124             HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes())) {
2125       return HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
2126     }
2127     CatalogTracker ct = getCatalogTracker();
2128     byte[] tmp = regionNameOrEncodedRegionName;
2129     try {
2130       Pair<HRegionInfo, ServerName> regionServerPair
2131         = getRegion(regionNameOrEncodedRegionName, ct);
2132       if (regionServerPair != null && regionServerPair.getFirst() != null) {
2133         tmp = regionServerPair.getFirst().getRegionName();
2134       }
2135     } finally {
2136       cleanupCatalogTracker(ct);
2137     }
2138     return tmp;
2139   }
2140 
2141   /**
2142    * Check if table exists or not
2143    * @param tableName Name of a table.
2144    * @param ct A {@link CatalogTracker} instance (caller of this method usually has one).
2145    * @return tableName instance
2146    * @throws IOException if a remote or network exception occurs.
2147    * @throws TableNotFoundException if table does not exist.
2148    */
2149   //TODO rename this method
2150   private TableName checkTableExists(
2151       final TableName tableName, CatalogTracker ct)
2152       throws IOException {
2153     if (!MetaReader.tableExists(ct, tableName)) {
2154       throw new TableNotFoundException(tableName);
2155     }
2156     return tableName;
2157   }
2158 
2159   /**
2160    * Shuts down the HBase cluster
2161    * @throws IOException if a remote or network exception occurs
2162    */
2163   public synchronized void shutdown() throws IOException {
2164     executeCallable(new MasterCallable<Void>(getConnection()) {
2165       @Override
2166       public Void call() throws ServiceException {
2167         master.shutdown(null,ShutdownRequest.newBuilder().build());
2168         return null;
2169       }
2170     });
2171   }
2172 
2173   /**
2174    * Shuts down the current HBase master only.
2175    * Does not shutdown the cluster.
2176    * @see #shutdown()
2177    * @throws IOException if a remote or network exception occurs
2178    */
2179   public synchronized void stopMaster() throws IOException {
2180     executeCallable(new MasterCallable<Void>(getConnection()) {
2181       @Override
2182       public Void call() throws ServiceException {
2183         master.stopMaster(null,StopMasterRequest.newBuilder().build());
2184         return null;
2185       }
2186     });
2187   }
2188 
2189   /**
2190    * Stop the designated regionserver
2191    * @param hostnamePort Hostname and port delimited by a <code>:</code> as in
2192    * <code>example.org:1234</code>
2193    * @throws IOException if a remote or network exception occurs
2194    */
2195   public synchronized void stopRegionServer(final String hostnamePort)
2196   throws IOException {
2197     String hostname = Addressing.parseHostname(hostnamePort);
2198     int port = Addressing.parsePort(hostnamePort);
2199     AdminService.BlockingInterface admin =
2200       this.connection.getAdmin(ServerName.valueOf(hostname, port, 0));
2201     StopServerRequest request = RequestConverter.buildStopServerRequest(
2202       "Called by admin client " + this.connection.toString());
2203     try {
2204       admin.stopServer(null, request);
2205     } catch (ServiceException se) {
2206       throw ProtobufUtil.getRemoteException(se);
2207     }
2208   }
2209 
2210 
2211   /**
2212    * @return cluster status
2213    * @throws IOException if a remote or network exception occurs
2214    */
2215   public ClusterStatus getClusterStatus() throws IOException {
2216     return executeCallable(new MasterCallable<ClusterStatus>(getConnection()) {
2217       @Override
2218       public ClusterStatus call() throws ServiceException {
2219         GetClusterStatusRequest req = RequestConverter.buildGetClusterStatusRequest();
2220         return ClusterStatus.convert(master.getClusterStatus(null,req).getClusterStatus());
2221       }
2222     });
2223   }
2224 
2225   private HRegionLocation getFirstMetaServerForTable(final TableName tableName)
2226   throws IOException {
2227     return connection.locateRegion(TableName.META_TABLE_NAME,
2228       HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
2229   }
2230 
2231   /**
2232    * @return Configuration used by the instance.
2233    */
2234   public Configuration getConfiguration() {
2235     return this.conf;
2236   }
2237 
2238   /**
2239    * Create a new namespace
2240    * @param descriptor descriptor which describes the new namespace
2241    * @throws IOException
2242    */
2243   public void createNamespace(final NamespaceDescriptor descriptor) throws IOException {
2244     executeCallable(new MasterCallable<Void>(getConnection()) {
2245       @Override
2246       public Void call() throws Exception {
2247         master.createNamespace(null,
2248           CreateNamespaceRequest.newBuilder()
2249                 .setNamespaceDescriptor(ProtobufUtil
2250                     .toProtoNamespaceDescriptor(descriptor)).build());
2251         return null;
2252       }
2253     });
2254   }
2255 
2256   /**
2257    * Modify an existing namespace
2258    * @param descriptor descriptor which describes the new namespace
2259    * @throws IOException
2260    */
2261   public void modifyNamespace(final NamespaceDescriptor descriptor) throws IOException {
2262     executeCallable(new MasterCallable<Void>(getConnection()) {
2263       @Override
2264       public Void call() throws Exception {
2265         master.modifyNamespace(null, ModifyNamespaceRequest.newBuilder().
2266           setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(descriptor)).build());
2267         return null;
2268       }
2269     });
2270   }
2271 
2272   /**
2273    * Delete an existing namespace. Only empty namespaces (no tables) can be removed.
2274    * @param name namespace name
2275    * @throws IOException
2276    */
2277   public void deleteNamespace(final String name) throws IOException {
2278     executeCallable(new MasterCallable<Void>(getConnection()) {
2279       @Override
2280       public Void call() throws Exception {
2281         master.deleteNamespace(null, DeleteNamespaceRequest.newBuilder().
2282           setNamespaceName(name).build());
2283         return null;
2284       }
2285     });
2286   }
2287 
2288   /**
2289    * Get a namespace descriptor by name
2290    * @param name name of namespace descriptor
2291    * @return A descriptor
2292    * @throws IOException
2293    */
2294   public NamespaceDescriptor getNamespaceDescriptor(final String name) throws IOException {
2295     return
2296         executeCallable(new MasterCallable<NamespaceDescriptor>(getConnection()) {
2297           @Override
2298           public NamespaceDescriptor call() throws Exception {
2299             return ProtobufUtil.toNamespaceDescriptor(
2300               master.getNamespaceDescriptor(null, GetNamespaceDescriptorRequest.newBuilder().
2301                 setNamespaceName(name).build()).getNamespaceDescriptor());
2302           }
2303         });
2304   }
2305 
2306   /**
2307    * List available namespace descriptors
2308    * @return List of descriptors
2309    * @throws IOException
2310    */
2311   public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException {
2312     return
2313         executeCallable(new MasterCallable<NamespaceDescriptor[]>(getConnection()) {
2314           @Override
2315           public NamespaceDescriptor[] call() throws Exception {
2316             List<HBaseProtos.NamespaceDescriptor> list =
2317               master.listNamespaceDescriptors(null, ListNamespaceDescriptorsRequest.newBuilder().
2318                 build()).getNamespaceDescriptorList();
2319             NamespaceDescriptor[] res = new NamespaceDescriptor[list.size()];
2320             for(int i = 0; i < list.size(); i++) {
2321               res[i] = ProtobufUtil.toNamespaceDescriptor(list.get(i));
2322             }
2323             return res;
2324           }
2325         });
2326   }
2327 
2328   /**
2329    * Get list of table descriptors by namespace
2330    * @param name namespace name
2331    * @return A descriptor
2332    * @throws IOException
2333    */
2334   public HTableDescriptor[] listTableDescriptorsByNamespace(final String name) throws IOException {
2335     return
2336         executeCallable(new MasterCallable<HTableDescriptor[]>(getConnection()) {
2337           @Override
2338           public HTableDescriptor[] call() throws Exception {
2339             List<TableSchema> list =
2340               master.listTableDescriptorsByNamespace(null, ListTableDescriptorsByNamespaceRequest.
2341                 newBuilder().setNamespaceName(name).build()).getTableSchemaList();
2342             HTableDescriptor[] res = new HTableDescriptor[list.size()];
2343             for(int i=0; i < list.size(); i++) {
2344 
2345               res[i] = HTableDescriptor.convert(list.get(i));
2346             }
2347             return res;
2348           }
2349         });
2350   }
2351 
2352   /**
2353    * Get list of table names by namespace
2354    * @param name namespace name
2355    * @return The list of table names in the namespace
2356    * @throws IOException
2357    */
2358   public TableName[] listTableNamesByNamespace(final String name) throws IOException {
2359     return
2360         executeCallable(new MasterCallable<TableName[]>(getConnection()) {
2361           @Override
2362           public TableName[] call() throws Exception {
2363             List<HBaseProtos.TableName> tableNames =
2364               master.listTableNamesByNamespace(null, ListTableNamesByNamespaceRequest.
2365                 newBuilder().setNamespaceName(name).build())
2366                 .getTableNameList();
2367             TableName[] result = new TableName[tableNames.size()];
2368             for (int i = 0; i < tableNames.size(); i++) {
2369               result[i] = ProtobufUtil.toTableName(tableNames.get(i));
2370             }
2371             return result;
2372           }
2373         });
2374   }
2375 
2376   /**
2377    * Check to see if HBase is running. Throw an exception if not.
2378    * We consider that HBase is running if ZooKeeper and Master are running.
2379    *
2380    * @param conf system configuration
2381    * @throws MasterNotRunningException if the master is not running
2382    * @throws ZooKeeperConnectionException if unable to connect to zookeeper
2383    */
2384   public static void checkHBaseAvailable(Configuration conf)
2385     throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException {
2386     Configuration copyOfConf = HBaseConfiguration.create(conf);
2387 
2388     // We set it to make it fail as soon as possible if HBase is not available
2389     copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
2390     copyOfConf.setInt("zookeeper.recovery.retry", 0);
2391 
2392     HConnectionManager.HConnectionImplementation connection
2393       = (HConnectionManager.HConnectionImplementation)
2394       HConnectionManager.getConnection(copyOfConf);
2395 
2396     try {
2397       // Check ZK first.
2398       // If the connection exists, we may have a connection to ZK that does
2399       //  not work anymore
2400       ZooKeeperKeepAliveConnection zkw = null;
2401       try {
2402         zkw = connection.getKeepAliveZooKeeperWatcher();
2403         zkw.getRecoverableZooKeeper().getZooKeeper().exists(
2404           zkw.baseZNode, false);
2405 
2406       } catch (IOException e) {
2407         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
2408       } catch (InterruptedException e) {
2409         throw (InterruptedIOException)
2410             new InterruptedIOException("Can't connect to ZooKeeper").initCause(e);
2411       } catch (KeeperException e) {
2412         throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e);
2413       } finally {
2414         if (zkw != null) {
2415           zkw.close();
2416         }
2417       }
2418 
2419       // Check Master
2420       connection.isMasterRunning();
2421 
2422     } finally {
2423       connection.close();
2424     }
2425   }
2426 
2427   /**
2428    * get the regions of a given table.
2429    *
2430    * @param tableName the name of the table
2431    * @return Ordered list of {@link HRegionInfo}.
2432    * @throws IOException
2433    */
2434   public List<HRegionInfo> getTableRegions(final TableName tableName)
2435   throws IOException {
2436     CatalogTracker ct = getCatalogTracker();
2437     List<HRegionInfo> Regions = null;
2438     try {
2439       Regions = MetaReader.getTableRegions(ct, tableName, true);
2440     } finally {
2441       cleanupCatalogTracker(ct);
2442     }
2443     return Regions;
2444   }
2445 
2446   public List<HRegionInfo> getTableRegions(final byte[] tableName)
2447   throws IOException {
2448     return getTableRegions(TableName.valueOf(tableName));
2449   }
2450 
2451   @Override
2452   public synchronized void close() throws IOException {
2453     if (cleanupConnectionOnClose && this.connection != null && !this.closed) {
2454       this.connection.close();
2455       this.closed = true;
2456     }
2457   }
2458 
2459   /**
2460    * Get tableDescriptors
2461    * @param tableNames List of table names
2462    * @return HTD[] the tableDescriptor
2463    * @throws IOException if a remote or network exception occurs
2464    */
2465   public HTableDescriptor[] getTableDescriptorsByTableName(List<TableName> tableNames)
2466   throws IOException {
2467     return this.connection.getHTableDescriptorsByTableName(tableNames);
2468   }
2469 
2470   /**
2471    * Get tableDescriptors
2472    * @param names List of table names
2473    * @return HTD[] the tableDescriptor
2474    * @throws IOException if a remote or network exception occurs
2475    */
2476   public HTableDescriptor[] getTableDescriptors(List<String> names)
2477   throws IOException {
2478     List<TableName> tableNames = new ArrayList<TableName>(names.size());
2479     for(String name : names) {
2480       tableNames.add(TableName.valueOf(name));
2481     }
2482     return getTableDescriptorsByTableName(tableNames);
2483   }
2484 
2485   /**
2486    * Roll the log writer. That is, start writing log messages to a new file.
2487    *
2488    * @param serverName
2489    *          The servername of the regionserver. A server name is made of host,
2490    *          port and startcode. This is mandatory. Here is an example:
2491    *          <code> host187.example.com,60020,1289493121758</code>
2492    * @return If lots of logs, flush the returned regions so next time through
2493    * we can clean logs. Returns null if nothing to flush.  Names are actual
2494    * region names as returned by {@link HRegionInfo#getEncodedName()}
2495    * @throws IOException if a remote or network exception occurs
2496    * @throws FailedLogCloseException
2497    */
2498  public synchronized  byte[][] rollHLogWriter(String serverName)
2499       throws IOException, FailedLogCloseException {
2500     ServerName sn = ServerName.valueOf(serverName);
2501     AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2502     RollWALWriterRequest request = RequestConverter.buildRollWALWriterRequest();
2503     try {
2504       RollWALWriterResponse response = admin.rollWALWriter(null, request);
2505       int regionCount = response.getRegionToFlushCount();
2506       byte[][] regionsToFlush = new byte[regionCount][];
2507       for (int i = 0; i < regionCount; i++) {
2508         ByteString region = response.getRegionToFlush(i);
2509         regionsToFlush[i] = region.toByteArray();
2510       }
2511       return regionsToFlush;
2512     } catch (ServiceException se) {
2513       throw ProtobufUtil.getRemoteException(se);
2514     }
2515   }
2516 
2517   public String[] getMasterCoprocessors() {
2518     try {
2519       return getClusterStatus().getMasterCoprocessors();
2520     } catch (IOException e) {
2521       LOG.error("Could not getClusterStatus()",e);
2522       return null;
2523     }
2524   }
2525 
2526   /**
2527    * Get the current compaction state of a table or region.
2528    * It could be in a major compaction, a minor compaction, both, or none.
2529    *
2530    * @param tableNameOrRegionName table or region to major compact
2531    * @throws IOException if a remote or network exception occurs
2532    * @throws InterruptedException
2533    * @return the current compaction state
2534    */
2535   public CompactionState getCompactionState(final String tableNameOrRegionName)
2536       throws IOException, InterruptedException {
2537     return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
2538   }
2539 
2540   /**
2541    * Get the current compaction state of a table or region.
2542    * It could be in a major compaction, a minor compaction, both, or none.
2543    *
2544    * @param tableNameOrRegionName table or region to major compact
2545    * @throws IOException if a remote or network exception occurs
2546    * @throws InterruptedException
2547    * @return the current compaction state
2548    */
2549   public CompactionState getCompactionState(final byte[] tableNameOrRegionName)
2550       throws IOException, InterruptedException {
2551     CompactionState state = CompactionState.NONE;
2552     CatalogTracker ct = getCatalogTracker();
2553     try {
2554       Pair<HRegionInfo, ServerName> regionServerPair
2555         = getRegion(tableNameOrRegionName, ct);
2556       if (regionServerPair != null) {
2557         if (regionServerPair.getSecond() == null) {
2558           throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
2559         } else {
2560           ServerName sn = regionServerPair.getSecond();
2561           AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2562           GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2563             regionServerPair.getFirst().getRegionName(), true);
2564           GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2565           return response.getCompactionState();
2566         }
2567       } else {
2568         final TableName tableName =
2569             checkTableExists(TableName.valueOf(tableNameOrRegionName), ct);
2570         List<Pair<HRegionInfo, ServerName>> pairs =
2571           MetaReader.getTableRegionsAndLocations(ct, tableName);
2572         for (Pair<HRegionInfo, ServerName> pair: pairs) {
2573           if (pair.getFirst().isOffline()) continue;
2574           if (pair.getSecond() == null) continue;
2575           try {
2576             ServerName sn = pair.getSecond();
2577             AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
2578             GetRegionInfoRequest request = RequestConverter.buildGetRegionInfoRequest(
2579               pair.getFirst().getRegionName(), true);
2580             GetRegionInfoResponse response = admin.getRegionInfo(null, request);
2581             switch (response.getCompactionState()) {
2582             case MAJOR_AND_MINOR:
2583               return CompactionState.MAJOR_AND_MINOR;
2584             case MAJOR:
2585               if (state == CompactionState.MINOR) {
2586                 return CompactionState.MAJOR_AND_MINOR;
2587               }
2588               state = CompactionState.MAJOR;
2589               break;
2590             case MINOR:
2591               if (state == CompactionState.MAJOR) {
2592                 return CompactionState.MAJOR_AND_MINOR;
2593               }
2594               state = CompactionState.MINOR;
2595               break;
2596             case NONE:
2597               default: // nothing, continue
2598             }
2599           } catch (NotServingRegionException e) {
2600             if (LOG.isDebugEnabled()) {
2601               LOG.debug("Trying to get compaction state of " +
2602                 pair.getFirst() + ": " +
2603                 StringUtils.stringifyException(e));
2604             }
2605           } catch (RemoteException e) {
2606             if (e.getMessage().indexOf(NotServingRegionException.class.getName()) >= 0) {
2607               if (LOG.isDebugEnabled()) {
2608                 LOG.debug("Trying to get compaction state of " + pair.getFirst() + ": "
2609                     + StringUtils.stringifyException(e));
2610               }
2611             } else {
2612               throw e;
2613             }
2614           }
2615         }
2616       }
2617     } catch (ServiceException se) {
2618       throw ProtobufUtil.getRemoteException(se);
2619     } finally {
2620       cleanupCatalogTracker(ct);
2621     }
2622     return state;
2623   }
2624 
2625   /**
2626    * Take a snapshot for the given table. If the table is enabled, a FLUSH-type snapshot will be
2627    * taken. If the table is disabled, an offline snapshot is taken.
2628    * <p>
2629    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2630    * snapshot with the same name (even a different type or with different parameters) will fail with
2631    * a {@link SnapshotCreationException} indicating the duplicate naming.
2632    * <p>
2633    * Snapshot names follow the same naming constraints as tables in HBase. See
2634    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2635    * @param snapshotName name of the snapshot to be created
2636    * @param tableName name of the table for which snapshot is created
2637    * @throws IOException if a remote or network exception occurs
2638    * @throws SnapshotCreationException if snapshot creation failed
2639    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2640    */
2641   public void snapshot(final String snapshotName,
2642                        final TableName tableName) throws IOException,
2643       SnapshotCreationException, IllegalArgumentException {
2644     snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
2645   }
2646 
2647   public void snapshot(final String snapshotName,
2648                        final String tableName) throws IOException,
2649       SnapshotCreationException, IllegalArgumentException {
2650     snapshot(snapshotName, TableName.valueOf(tableName),
2651         SnapshotDescription.Type.FLUSH);
2652   }
2653 
2654   /**
2655    * Create snapshot for the given table of given flush type.
2656    * <p>
2657    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2658    * snapshot with the same name (even a different type or with different parameters) will fail with
2659    * a {@link SnapshotCreationException} indicating the duplicate naming.
2660    * <p>
2661    * Snapshot names follow the same naming constraints as tables in HBase.
2662    * @param snapshotName name of the snapshot to be created
2663    * @param tableName name of the table for which snapshot is created
2664    * @param flushType if the snapshot should be taken without flush memstore first
2665    * @throws IOException if a remote or network exception occurs
2666    * @throws SnapshotCreationException if snapshot creation failed
2667    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2668    */
2669    public void snapshot(final byte[] snapshotName, final byte[] tableName,
2670                        final SnapshotDescription.Type flushType) throws
2671       IOException, SnapshotCreationException, IllegalArgumentException {
2672       snapshot(Bytes.toString(snapshotName), Bytes.toString(tableName), flushType);
2673   }
2674   /**
2675    public void snapshot(final String snapshotName,
2676     * Create a timestamp consistent snapshot for the given table.
2677                         final byte[] tableName) throws IOException,
2678     * <p>
2679     * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2680     * snapshot with the same name (even a different type or with different parameters) will fail with
2681     * a {@link SnapshotCreationException} indicating the duplicate naming.
2682     * <p>
2683     * Snapshot names follow the same naming constraints as tables in HBase.
2684     * @param snapshotName name of the snapshot to be created
2685     * @param tableName name of the table for which snapshot is created
2686     * @throws IOException if a remote or network exception occurs
2687     * @throws SnapshotCreationException if snapshot creation failed
2688     * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2689     */
2690   public void snapshot(final byte[] snapshotName,
2691                        final TableName tableName) throws IOException,
2692       SnapshotCreationException, IllegalArgumentException {
2693     snapshot(Bytes.toString(snapshotName), tableName, SnapshotDescription.Type.FLUSH);
2694   }
2695 
2696   public void snapshot(final byte[] snapshotName,
2697                        final byte[] tableName) throws IOException,
2698       SnapshotCreationException, IllegalArgumentException {
2699     snapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName),
2700         SnapshotDescription.Type.FLUSH);
2701   }
2702 
2703   /**
2704    * Create typed snapshot of the table.
2705    * <p>
2706    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2707    * snapshot with the same name (even a different type or with different parameters) will fail with
2708    * a {@link SnapshotCreationException} indicating the duplicate naming.
2709    * <p>
2710    * Snapshot names follow the same naming constraints as tables in HBase. See
2711    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2712    * <p>
2713    * @param snapshotName name to give the snapshot on the filesystem. Must be unique from all other
2714    *          snapshots stored on the cluster
2715    * @param tableName name of the table to snapshot
2716    * @param type type of snapshot to take
2717    * @throws IOException we fail to reach the master
2718    * @throws SnapshotCreationException if snapshot creation failed
2719    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2720    */
2721   public void snapshot(final String snapshotName,
2722                        final TableName tableName,
2723                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2724       IllegalArgumentException {
2725     SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
2726     builder.setTable(tableName.getNameAsString());
2727     builder.setName(snapshotName);
2728     builder.setType(type);
2729     snapshot(builder.build());
2730   }
2731 
2732   public void snapshot(final String snapshotName,
2733                        final String tableName,
2734                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2735       IllegalArgumentException {
2736     snapshot(snapshotName, TableName.valueOf(tableName), type);
2737   }
2738 
2739   public void snapshot(final String snapshotName,
2740                        final byte[] tableName,
2741                       SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
2742       IllegalArgumentException {
2743     snapshot(snapshotName, TableName.valueOf(tableName), type);
2744   }
2745 
2746   /**
2747    * Take a snapshot and wait for the server to complete that snapshot (blocking).
2748    * <p>
2749    * Only a single snapshot should be taken at a time for an instance of HBase, or results may be
2750    * undefined (you can tell multiple HBase clusters to snapshot at the same time, but only one at a
2751    * time for a single cluster).
2752    * <p>
2753    * Snapshots are considered unique based on <b>the name of the snapshot</b>. Attempts to take a
2754    * snapshot with the same name (even a different type or with different parameters) will fail with
2755    * a {@link SnapshotCreationException} indicating the duplicate naming.
2756    * <p>
2757    * Snapshot names follow the same naming constraints as tables in HBase. See
2758    * {@link org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
2759    * <p>
2760    * You should probably use {@link #snapshot(String, String)} or {@link #snapshot(byte[], byte[])}
2761    * unless you are sure about the type of snapshot that you want to take.
2762    * @param snapshot snapshot to take
2763    * @throws IOException or we lose contact with the master.
2764    * @throws SnapshotCreationException if snapshot failed to be taken
2765    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2766    */
2767   public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException,
2768       IllegalArgumentException {
2769     // actually take the snapshot
2770     SnapshotResponse response = takeSnapshotAsync(snapshot);
2771     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot)
2772         .build();
2773     IsSnapshotDoneResponse done = null;
2774     long start = EnvironmentEdgeManager.currentTimeMillis();
2775     long max = response.getExpectedTimeout();
2776     long maxPauseTime = max / this.numRetries;
2777     int tries = 0;
2778     LOG.debug("Waiting a max of " + max + " ms for snapshot '" +
2779         ClientSnapshotDescriptionUtils.toString(snapshot) + "'' to complete. (max " +
2780         maxPauseTime + " ms per retry)");
2781     while (tries == 0
2782         || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done.getDone())) {
2783       try {
2784         // sleep a backoff <= pauseTime amount
2785         long sleep = getPauseTime(tries++);
2786         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
2787         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
2788           "ms while waiting for snapshot completion.");
2789         Thread.sleep(sleep);
2790 
2791       } catch (InterruptedException e) {
2792         LOG.debug("Interrupted while waiting for snapshot " + snapshot + " to complete");
2793         Thread.currentThread().interrupt();
2794       }
2795       LOG.debug("Getting current status of snapshot from master...");
2796       done = executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2797         @Override
2798         public IsSnapshotDoneResponse call() throws ServiceException {
2799           return master.isSnapshotDone(null, request);
2800         }
2801       });
2802     };
2803     if (!done.getDone()) {
2804       throw new SnapshotCreationException("Snapshot '" + snapshot.getName()
2805           + "' wasn't completed in expectedTime:" + max + " ms", snapshot);
2806     }
2807   }
2808 
2809   /**
2810    * Take a snapshot without waiting for the server to complete that snapshot (asynchronous)
2811    * <p>
2812    * Only a single snapshot should be taken at a time, or results may be undefined.
2813    * @param snapshot snapshot to take
2814    * @return response from the server indicating the max time to wait for the snapshot
2815    * @throws IOException if the snapshot did not succeed or we lose contact with the master.
2816    * @throws SnapshotCreationException if snapshot creation failed
2817    * @throws IllegalArgumentException if the snapshot request is formatted incorrectly
2818    */
2819   public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException,
2820       SnapshotCreationException {
2821     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
2822     final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot)
2823         .build();
2824     // run the snapshot on the master
2825     return executeCallable(new MasterCallable<SnapshotResponse>(getConnection()) {
2826       @Override
2827       public SnapshotResponse call() throws ServiceException {
2828         return master.snapshot(null, request);
2829       }
2830     });
2831   }
2832 
2833   /**
2834    * Check the current state of the passed snapshot.
2835    * <p>
2836    * There are three possible states:
2837    * <ol>
2838    * <li>running - returns <tt>false</tt></li>
2839    * <li>finished - returns <tt>true</tt></li>
2840    * <li>finished with error - throws the exception that caused the snapshot to fail</li>
2841    * </ol>
2842    * <p>
2843    * The cluster only knows about the most recent snapshot. Therefore, if another snapshot has been
2844    * run/started since the snapshot your are checking, you will recieve an
2845    * {@link UnknownSnapshotException}.
2846    * @param snapshot description of the snapshot to check
2847    * @return <tt>true</tt> if the snapshot is completed, <tt>false</tt> if the snapshot is still
2848    *         running
2849    * @throws IOException if we have a network issue
2850    * @throws HBaseSnapshotException if the snapshot failed
2851    * @throws UnknownSnapshotException if the requested snapshot is unknown
2852    */
2853   public boolean isSnapshotFinished(final SnapshotDescription snapshot)
2854       throws IOException, HBaseSnapshotException, UnknownSnapshotException {
2855 
2856     return executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
2857       @Override
2858       public IsSnapshotDoneResponse call() throws ServiceException {
2859         return master.isSnapshotDone(null,
2860           IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build());
2861       }
2862     }).getDone();
2863   }
2864 
2865   /**
2866    * Restore the specified snapshot on the original table. (The table must be disabled)
2867    * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
2868    * is set to true, a snapshot of the current table is taken
2869    * before executing the restore operation.
2870    * In case of restore failure, the failsafe snapshot will be restored.
2871    * If the restore completes without problem the failsafe snapshot is deleted.
2872    *
2873    * @param snapshotName name of the snapshot to restore
2874    * @throws IOException if a remote or network exception occurs
2875    * @throws RestoreSnapshotException if snapshot failed to be restored
2876    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2877    */
2878   public void restoreSnapshot(final byte[] snapshotName)
2879       throws IOException, RestoreSnapshotException {
2880     restoreSnapshot(Bytes.toString(snapshotName));
2881   }
2882 
2883   /**
2884    * Restore the specified snapshot on the original table. (The table must be disabled)
2885    * If the "hbase.snapshot.restore.take.failsafe.snapshot" configuration property
2886    * is set to true, a snapshot of the current table is taken
2887    * before executing the restore operation.
2888    * In case of restore failure, the failsafe snapshot will be restored.
2889    * If the restore completes without problem the failsafe snapshot is deleted.
2890    *
2891    * @param snapshotName name of the snapshot to restore
2892    * @throws IOException if a remote or network exception occurs
2893    * @throws RestoreSnapshotException if snapshot failed to be restored
2894    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2895    */
2896   public void restoreSnapshot(final String snapshotName)
2897       throws IOException, RestoreSnapshotException {
2898     boolean takeFailSafeSnapshot =
2899       conf.getBoolean("hbase.snapshot.restore.take.failsafe.snapshot", false);
2900     restoreSnapshot(snapshotName, takeFailSafeSnapshot);
2901   }
2902 
2903   /**
2904    * Restore the specified snapshot on the original table. (The table must be disabled)
2905    * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
2906    * before executing the restore operation.
2907    * In case of restore failure, the failsafe snapshot will be restored.
2908    * If the restore completes without problem the failsafe snapshot is deleted.
2909    *
2910    * The failsafe snapshot name is configurable by using the property
2911    * "hbase.snapshot.restore.failsafe.name".
2912    *
2913    * @param snapshotName name of the snapshot to restore
2914    * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
2915    * @throws IOException if a remote or network exception occurs
2916    * @throws RestoreSnapshotException if snapshot failed to be restored
2917    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2918    */
2919   public void restoreSnapshot(final byte[] snapshotName, final boolean takeFailSafeSnapshot)
2920       throws IOException, RestoreSnapshotException {
2921     restoreSnapshot(Bytes.toString(snapshotName), takeFailSafeSnapshot);
2922   }
2923 
2924   /**
2925    * Restore the specified snapshot on the original table. (The table must be disabled)
2926    * If 'takeFailSafeSnapshot' is set to true, a snapshot of the current table is taken
2927    * before executing the restore operation.
2928    * In case of restore failure, the failsafe snapshot will be restored.
2929    * If the restore completes without problem the failsafe snapshot is deleted.
2930    *
2931    * The failsafe snapshot name is configurable by using the property
2932    * "hbase.snapshot.restore.failsafe.name".
2933    *
2934    * @param snapshotName name of the snapshot to restore
2935    * @param takeFailSafeSnapshot true if the failsafe snapshot should be taken
2936    * @throws IOException if a remote or network exception occurs
2937    * @throws RestoreSnapshotException if snapshot failed to be restored
2938    * @throws IllegalArgumentException if the restore request is formatted incorrectly
2939    */
2940   public void restoreSnapshot(final String snapshotName, boolean takeFailSafeSnapshot)
2941       throws IOException, RestoreSnapshotException {
2942     TableName tableName = null;
2943     for (SnapshotDescription snapshotInfo: listSnapshots()) {
2944       if (snapshotInfo.getName().equals(snapshotName)) {
2945         tableName = TableName.valueOf(snapshotInfo.getTable());
2946         break;
2947       }
2948     }
2949 
2950     if (tableName == null) {
2951       throw new RestoreSnapshotException(
2952         "Unable to find the table name for snapshot=" + snapshotName);
2953     }
2954 
2955     // The table does not exists, switch to clone.
2956     if (!tableExists(tableName)) {
2957       try {
2958         cloneSnapshot(snapshotName, tableName);
2959       } catch (InterruptedException e) {
2960         throw new InterruptedIOException("Interrupted when restoring a nonexistent table: " +
2961           e.getMessage());
2962       }
2963       return;
2964     }
2965 
2966     // Check if the table is disabled
2967     if (!isTableDisabled(tableName)) {
2968       throw new TableNotDisabledException(tableName);
2969     }
2970 
2971     // Take a snapshot of the current state
2972     String failSafeSnapshotSnapshotName = null;
2973     if (takeFailSafeSnapshot) {
2974       failSafeSnapshotSnapshotName = conf.get("hbase.snapshot.restore.failsafe.name",
2975         "hbase-failsafe-{snapshot.name}-{restore.timestamp}");
2976       failSafeSnapshotSnapshotName = failSafeSnapshotSnapshotName
2977         .replace("{snapshot.name}", snapshotName)
2978         .replace("{table.name}", tableName.toString().replace(TableName.NAMESPACE_DELIM, '.'))
2979         .replace("{restore.timestamp}", String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
2980       LOG.info("Taking restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
2981       snapshot(failSafeSnapshotSnapshotName, tableName);
2982     }
2983 
2984     try {
2985       // Restore snapshot
2986       internalRestoreSnapshot(snapshotName, tableName);
2987     } catch (IOException e) {
2988       // Somthing went wrong during the restore...
2989       // if the pre-restore snapshot is available try to rollback
2990       if (takeFailSafeSnapshot) {
2991         try {
2992           internalRestoreSnapshot(failSafeSnapshotSnapshotName, tableName);
2993           String msg = "Restore snapshot=" + snapshotName +
2994             " failed. Rollback to snapshot=" + failSafeSnapshotSnapshotName + " succeeded.";
2995           LOG.error(msg, e);
2996           throw new RestoreSnapshotException(msg, e);
2997         } catch (IOException ex) {
2998           String msg = "Failed to restore and rollback to snapshot=" + failSafeSnapshotSnapshotName;
2999           LOG.error(msg, ex);
3000           throw new RestoreSnapshotException(msg, e);
3001         }
3002       } else {
3003         throw new RestoreSnapshotException("Failed to restore snapshot=" + snapshotName, e);
3004       }
3005     }
3006 
3007     // If the restore is succeeded, delete the pre-restore snapshot
3008     if (takeFailSafeSnapshot) {
3009       try {
3010         LOG.info("Deleting restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
3011         deleteSnapshot(failSafeSnapshotSnapshotName);
3012       } catch (IOException e) {
3013         LOG.error("Unable to remove the failsafe snapshot: " + failSafeSnapshotSnapshotName, e);
3014       }
3015     }
3016   }
3017 
3018   /**
3019    * Create a new table by cloning the snapshot content.
3020    *
3021    * @param snapshotName name of the snapshot to be cloned
3022    * @param tableName name of the table where the snapshot will be restored
3023    * @throws IOException if a remote or network exception occurs
3024    * @throws TableExistsException if table to be created already exists
3025    * @throws RestoreSnapshotException if snapshot failed to be cloned
3026    * @throws IllegalArgumentException if the specified table has not a valid name
3027    */
3028   public void cloneSnapshot(final byte[] snapshotName, final byte[] tableName)
3029       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3030     cloneSnapshot(Bytes.toString(snapshotName), TableName.valueOf(tableName));
3031   }
3032 
3033   /**
3034    * Create a new table by cloning the snapshot content.
3035    *
3036    * @param snapshotName name of the snapshot to be cloned
3037    * @param tableName name of the table where the snapshot will be restored
3038    * @throws IOException if a remote or network exception occurs
3039    * @throws TableExistsException if table to be created already exists
3040    * @throws RestoreSnapshotException if snapshot failed to be cloned
3041    * @throws IllegalArgumentException if the specified table has not a valid name
3042    */
3043   public void cloneSnapshot(final byte[] snapshotName, final TableName tableName)
3044       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3045     cloneSnapshot(Bytes.toString(snapshotName), tableName);
3046   }
3047 
3048 
3049 
3050   /**
3051    * Create a new table by cloning the snapshot content.
3052    *
3053    * @param snapshotName name of the snapshot to be cloned
3054    * @param tableName name of the table where the snapshot will be restored
3055    * @throws IOException if a remote or network exception occurs
3056    * @throws TableExistsException if table to be created already exists
3057    * @throws RestoreSnapshotException if snapshot failed to be cloned
3058    * @throws IllegalArgumentException if the specified table has not a valid name
3059    */
3060   public void cloneSnapshot(final String snapshotName, final String tableName)
3061       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3062     cloneSnapshot(snapshotName, TableName.valueOf(tableName));
3063   }
3064 
3065   /**
3066    * Create a new table by cloning the snapshot content.
3067    *
3068    * @param snapshotName name of the snapshot to be cloned
3069    * @param tableName name of the table where the snapshot will be restored
3070    * @throws IOException if a remote or network exception occurs
3071    * @throws TableExistsException if table to be created already exists
3072    * @throws RestoreSnapshotException if snapshot failed to be cloned
3073    * @throws IllegalArgumentException if the specified table has not a valid name
3074    */
3075   public void cloneSnapshot(final String snapshotName, final TableName tableName)
3076       throws IOException, TableExistsException, RestoreSnapshotException, InterruptedException {
3077     if (tableExists(tableName)) {
3078       throw new TableExistsException(tableName);
3079     }
3080     internalRestoreSnapshot(snapshotName, tableName);
3081     waitUntilTableIsEnabled(tableName);
3082   }
3083 
3084   /**
3085    * Execute a distributed procedure on a cluster.
3086    *
3087    * @param signature A distributed procedure is uniquely identified
3088    * by its signature (default the root ZK node name of the procedure).
3089    * @param instance The instance name of the procedure. For some procedures, this parameter is
3090    * optional.
3091    * @param props Property/Value pairs of properties passing to the procedure
3092    */
3093   public void execProcedure(String signature, String instance,
3094       Map<String, String> props) throws IOException {
3095     ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
3096     builder.setSignature(signature).setInstance(instance);
3097     for (String key : props.keySet()) {
3098       NameStringPair pair = NameStringPair.newBuilder().setName(key)
3099           .setValue(props.get(key)).build();
3100       builder.addConfiguration(pair);
3101     }
3102 
3103     final ExecProcedureRequest request = ExecProcedureRequest.newBuilder()
3104         .setProcedure(builder.build()).build();
3105     // run the procedure on the master
3106     ExecProcedureResponse response = executeCallable(new MasterCallable<ExecProcedureResponse>(
3107         getConnection()) {
3108       @Override
3109       public ExecProcedureResponse call() throws ServiceException {
3110         return master.execProcedure(null, request);
3111       }
3112     });
3113 
3114     long start = EnvironmentEdgeManager.currentTimeMillis();
3115     long max = response.getExpectedTimeout();
3116     long maxPauseTime = max / this.numRetries;
3117     int tries = 0;
3118     LOG.debug("Waiting a max of " + max + " ms for procedure '" +
3119         signature + " : " + instance + "'' to complete. (max " + maxPauseTime + " ms per retry)");
3120     boolean done = false;
3121     while (tries == 0
3122         || ((EnvironmentEdgeManager.currentTimeMillis() - start) < max && !done)) {
3123       try {
3124         // sleep a backoff <= pauseTime amount
3125         long sleep = getPauseTime(tries++);
3126         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
3127         LOG.debug("(#" + tries + ") Sleeping: " + sleep +
3128           "ms while waiting for procedure completion.");
3129         Thread.sleep(sleep);
3130 
3131       } catch (InterruptedException e) {
3132         LOG.debug("Interrupted while waiting for procedure " + signature + " to complete");
3133         Thread.currentThread().interrupt();
3134       }
3135       LOG.debug("Getting current status of procedure from master...");
3136       done = isProcedureFinished(signature, instance, props);
3137     }
3138     if (!done) {
3139       throw new IOException("Procedure '" + signature + " : " + instance
3140           + "' wasn't completed in expectedTime:" + max + " ms");
3141     }
3142   }
3143 
3144   /**
3145    * Check the current state of the specified procedure.
3146    * <p>
3147    * There are three possible states:
3148    * <ol>
3149    * <li>running - returns <tt>false</tt></li>
3150    * <li>finished - returns <tt>true</tt></li>
3151    * <li>finished with error - throws the exception that caused the procedure to fail</li>
3152    * </ol>
3153    * <p>
3154    *
3155    * @param signature The signature that uniquely identifies a procedure
3156    * @param instance The instance name of the procedure
3157    * @param props Property/Value pairs of properties passing to the procedure
3158    * @return true if the specified procedure is finished successfully, false if it is still running
3159    * @throws IOException if the specified procedure finished with error
3160    */
3161   public boolean isProcedureFinished(String signature, String instance, Map<String, String> props)
3162       throws IOException {
3163     final ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
3164     builder.setSignature(signature).setInstance(instance);
3165     for (String key : props.keySet()) {
3166       NameStringPair pair = NameStringPair.newBuilder().setName(key)
3167           .setValue(props.get(key)).build();
3168       builder.addConfiguration(pair);
3169     }
3170     final ProcedureDescription desc = builder.build();
3171     return executeCallable(
3172         new MasterCallable<IsProcedureDoneResponse>(getConnection()) {
3173           @Override
3174           public IsProcedureDoneResponse call() throws ServiceException {
3175             return master.isProcedureDone(null, IsProcedureDoneRequest
3176                 .newBuilder().setProcedure(desc).build());
3177           }
3178         }).getDone();
3179   }
3180 
3181   /**
3182    * Execute Restore/Clone snapshot and wait for the server to complete (blocking).
3183    * To check if the cloned table exists, use {@link #isTableAvailable} -- it is not safe to
3184    * create an HTable instance to this table before it is available.
3185    * @param snapshotName snapshot to restore
3186    * @param tableName table name to restore the snapshot on
3187    * @throws IOException if a remote or network exception occurs
3188    * @throws RestoreSnapshotException if snapshot failed to be restored
3189    * @throws IllegalArgumentException if the restore request is formatted incorrectly
3190    */
3191   private void internalRestoreSnapshot(final String snapshotName, final TableName
3192       tableName)
3193       throws IOException, RestoreSnapshotException {
3194     SnapshotDescription snapshot = SnapshotDescription.newBuilder()
3195         .setName(snapshotName).setTable(tableName.getNameAsString()).build();
3196 
3197     // actually restore the snapshot
3198     internalRestoreSnapshotAsync(snapshot);
3199 
3200     final IsRestoreSnapshotDoneRequest request = IsRestoreSnapshotDoneRequest.newBuilder()
3201         .setSnapshot(snapshot).build();
3202     IsRestoreSnapshotDoneResponse done = IsRestoreSnapshotDoneResponse.newBuilder()
3203         .setDone(false).buildPartial();
3204     final long maxPauseTime = 5000;
3205     int tries = 0;
3206     while (!done.getDone()) {
3207       try {
3208         // sleep a backoff <= pauseTime amount
3209         long sleep = getPauseTime(tries++);
3210         sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
3211         LOG.debug(tries + ") Sleeping: " + sleep + " ms while we wait for snapshot restore to complete.");
3212         Thread.sleep(sleep);
3213       } catch (InterruptedException e) {
3214         LOG.debug("Interrupted while waiting for snapshot " + snapshot + " restore to complete");
3215         Thread.currentThread().interrupt();
3216       }
3217       LOG.debug("Getting current status of snapshot restore from master...");
3218       done = executeCallable(new MasterCallable<IsRestoreSnapshotDoneResponse>(
3219           getConnection()) {
3220         @Override
3221         public IsRestoreSnapshotDoneResponse call() throws ServiceException {
3222           return master.isRestoreSnapshotDone(null, request);
3223         }
3224       });
3225     }
3226     if (!done.getDone()) {
3227       throw new RestoreSnapshotException("Snapshot '" + snapshot.getName() + "' wasn't restored.");
3228     }
3229   }
3230 
3231   /**
3232    * Execute Restore/Clone snapshot and wait for the server to complete (asynchronous)
3233    * <p>
3234    * Only a single snapshot should be restored at a time, or results may be undefined.
3235    * @param snapshot snapshot to restore
3236    * @return response from the server indicating the max time to wait for the snapshot
3237    * @throws IOException if a remote or network exception occurs
3238    * @throws RestoreSnapshotException if snapshot failed to be restored
3239    * @throws IllegalArgumentException if the restore request is formatted incorrectly
3240    */
3241   private RestoreSnapshotResponse internalRestoreSnapshotAsync(final SnapshotDescription snapshot)
3242       throws IOException, RestoreSnapshotException {
3243     ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
3244 
3245     final RestoreSnapshotRequest request = RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot)
3246         .build();
3247 
3248     // run the snapshot restore on the master
3249     return executeCallable(new MasterCallable<RestoreSnapshotResponse>(getConnection()) {
3250       @Override
3251       public RestoreSnapshotResponse call() throws ServiceException {
3252         return master.restoreSnapshot(null, request);
3253       }
3254     });
3255   }
3256 
3257   /**
3258    * List completed snapshots.
3259    * @return a list of snapshot descriptors for completed snapshots
3260    * @throws IOException if a network error occurs
3261    */
3262   public List<SnapshotDescription> listSnapshots() throws IOException {
3263     return executeCallable(new MasterCallable<List<SnapshotDescription>>(getConnection()) {
3264       @Override
3265       public List<SnapshotDescription> call() throws ServiceException {
3266         return master.getCompletedSnapshots(null, GetCompletedSnapshotsRequest.newBuilder().build())
3267             .getSnapshotsList();
3268       }
3269     });
3270   }
3271 
3272   /**
3273    * List all the completed snapshots matching the given regular expression.
3274    *
3275    * @param regex The regular expression to match against
3276    * @return - returns a List of SnapshotDescription
3277    * @throws IOException if a remote or network exception occurs
3278    */
3279   public List<SnapshotDescription> listSnapshots(String regex) throws IOException {
3280     return listSnapshots(Pattern.compile(regex));
3281   }
3282 
3283   /**
3284    * List all the completed snapshots matching the given pattern.
3285    *
3286    * @param pattern The compiled regular expression to match against
3287    * @return - returns a List of SnapshotDescription
3288    * @throws IOException if a remote or network exception occurs
3289    */
3290   public List<SnapshotDescription> listSnapshots(Pattern pattern) throws IOException {
3291     List<SnapshotDescription> matched = new LinkedList<SnapshotDescription>();
3292     List<SnapshotDescription> snapshots = listSnapshots();
3293     for (SnapshotDescription snapshot : snapshots) {
3294       if (pattern.matcher(snapshot.getName()).matches()) {
3295         matched.add(snapshot);
3296       }
3297     }
3298     return matched;
3299   }
3300 
3301   /**
3302    * List all the completed snapshots matching the given table name regular expression and snapshot
3303    * name regular expression.
3304    * @param tableNameRegex The table name regular expression to match against
3305    * @param snapshotNameRegex The snapshot name regular expression to match against
3306    * @return returns a List of completed SnapshotDescription
3307    * @throws IOException if a remote or network exception occurs
3308    */
3309   public List<SnapshotDescription> listTableSnapshots(String tableNameRegex,
3310       String snapshotNameRegex) throws IOException {
3311     return listTableSnapshots(Pattern.compile(tableNameRegex), Pattern.compile(snapshotNameRegex));
3312   }
3313 
3314   /**
3315    * List all the completed snapshots matching the given table name regular expression and snapshot
3316    * name regular expression.
3317    * @param tableNamePattern The compiled table name regular expression to match against
3318    * @param snapshotNamePattern The compiled snapshot name regular expression to match against
3319    * @return returns a List of completed SnapshotDescription
3320    * @throws IOException if a remote or network exception occurs
3321    */
3322   public List<SnapshotDescription> listTableSnapshots(Pattern tableNamePattern,
3323       Pattern snapshotNamePattern) throws IOException {
3324     String[] tableNames = getTableNames(tableNamePattern);
3325 
3326     List<SnapshotDescription> tableSnapshots = new LinkedList<SnapshotDescription>();
3327     List<SnapshotDescription> snapshots = listSnapshots(snapshotNamePattern);
3328 
3329     List<String> listOfTableNames = Arrays.asList(tableNames);
3330     for (SnapshotDescription snapshot : snapshots) {
3331       if (listOfTableNames.contains(snapshot.getTable())) {
3332         tableSnapshots.add(snapshot);
3333       }
3334     }
3335     return tableSnapshots;
3336   }
3337 
3338   /**
3339    * Delete an existing snapshot.
3340    * @param snapshotName name of the snapshot
3341    * @throws IOException if a remote or network exception occurs
3342    */
3343   public void deleteSnapshot(final byte[] snapshotName) throws IOException {
3344     deleteSnapshot(Bytes.toString(snapshotName));
3345   }
3346 
3347   /**
3348    * Delete an existing snapshot.
3349    * @param snapshotName name of the snapshot
3350    * @throws IOException if a remote or network exception occurs
3351    */
3352   public void deleteSnapshot(final String snapshotName) throws IOException {
3353     // make sure the snapshot is possibly valid
3354     TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(snapshotName));
3355     // do the delete
3356     executeCallable(new MasterCallable<Void>(getConnection()) {
3357       @Override
3358       public Void call() throws ServiceException {
3359         master.deleteSnapshot(null,
3360           DeleteSnapshotRequest.newBuilder().
3361             setSnapshot(SnapshotDescription.newBuilder().setName(snapshotName).build()).build());
3362         return null;
3363       }
3364     });
3365   }
3366 
3367   /**
3368    * Delete existing snapshots whose names match the pattern passed.
3369    * @param regex The regular expression to match against
3370    * @throws IOException if a remote or network exception occurs
3371    */
3372   public void deleteSnapshots(final String regex) throws IOException {
3373     deleteSnapshots(Pattern.compile(regex));
3374   }
3375 
3376   /**
3377    * Delete existing snapshots whose names match the pattern passed.
3378    * @param pattern pattern for names of the snapshot to match
3379    * @throws IOException if a remote or network exception occurs
3380    */
3381   public void deleteSnapshots(final Pattern pattern) throws IOException {
3382     List<SnapshotDescription> snapshots = listSnapshots(pattern);
3383     for (final SnapshotDescription snapshot : snapshots) {
3384        try {
3385          internalDeleteSnapshot(snapshot);
3386        } catch (IOException ex) {
3387          LOG.info(
3388            "Failed to delete snapshot " + snapshot.getName() + " for table " + snapshot.getTable(),
3389            ex);
3390        }
3391     }
3392   }
3393 
3394   /**
3395    * Delete all existing snapshots matching the given table name regular expression and snapshot
3396    * name regular expression.
3397    * @param tableNameRegex The table name regular expression to match against
3398    * @param snapshotNameRegex The snapshot name regular expression to match against
3399    * @throws IOException if a remote or network exception occurs
3400    */
3401   public void deleteTableSnapshots(String tableNameRegex, String snapshotNameRegex)
3402       throws IOException {
3403     deleteTableSnapshots(Pattern.compile(tableNameRegex), Pattern.compile(snapshotNameRegex));
3404   }
3405 
3406   /**
3407    * Delete all existing snapshots matching the given table name regular expression and snapshot
3408    * name regular expression.
3409    * @param tableNamePattern The compiled table name regular expression to match against
3410    * @param snapshotNamePattern The compiled snapshot name regular expression to match against
3411    * @throws IOException if a remote or network exception occurs
3412    */
3413   public void deleteTableSnapshots(Pattern tableNamePattern, Pattern snapshotNamePattern)
3414       throws IOException {
3415     List<SnapshotDescription> snapshots = listTableSnapshots(tableNamePattern, snapshotNamePattern);
3416     for (SnapshotDescription snapshot : snapshots) {
3417       try {
3418         internalDeleteSnapshot(snapshot);
3419         LOG.debug("Successfully deleted snapshot: " + snapshot.getName());
3420       } catch (IOException e) {
3421         LOG.error("Failed to delete snapshot: " + snapshot.getName(), e);
3422       }
3423     }
3424   }
3425 
3426   private void internalDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
3427     executeCallable(new MasterCallable<Void>(getConnection()) {
3428       @Override
3429       public Void call() throws ServiceException {
3430         this.master.deleteSnapshot(null, DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot)
3431             .build());
3432         return null;
3433       }
3434     });
3435   }
3436 
3437   /**
3438    * Parent of {@link MasterCallable} and {@link MasterCallable}.
3439    * Has common methods.
3440    * @param <V>
3441    */
3442   abstract static class MasterCallable<V> implements RetryingCallable<V>, Closeable {
3443     protected HConnection connection;
3444     protected MasterKeepAliveConnection master;
3445 
3446     public MasterCallable(final HConnection connection) {
3447       this.connection = connection;
3448     }
3449 
3450     @Override
3451     public void prepare(boolean reload) throws IOException {
3452       this.master = this.connection.getKeepAliveMasterService();
3453     }
3454 
3455     @Override
3456     public void close() throws IOException {
3457       // The above prepare could fail but this would still be called though masterAdmin is null
3458       if (this.master != null) this.master.close();
3459     }
3460 
3461     @Override
3462     public void throwable(Throwable t, boolean retrying) {
3463     }
3464 
3465     @Override
3466     public String getExceptionMessageAdditionalDetail() {
3467       return "";
3468     }
3469 
3470     @Override
3471     public long sleep(long pause, int tries) {
3472       return ConnectionUtils.getPauseTime(pause, tries);
3473     }
3474   }
3475 
3476   private <V> V executeCallable(MasterCallable<V> callable) throws IOException {
3477     RpcRetryingCaller<V> caller = rpcCallerFactory.newCaller();
3478     try {
3479       return caller.callWithRetries(callable);
3480     } finally {
3481       callable.close();
3482     }
3483   }
3484 
3485   /**
3486    * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
3487    * connected to the active master.
3488    *
3489    * <p>
3490    * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
3491    * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
3492    * </p>
3493    *
3494    * <div style="background-color: #cccccc; padding: 2px">
3495    * <blockquote><pre>
3496    * CoprocessorRpcChannel channel = myAdmin.coprocessorService();
3497    * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
3498    * MyCallRequest request = MyCallRequest.newBuilder()
3499    *     ...
3500    *     .build();
3501    * MyCallResponse response = service.myCall(null, request);
3502    * </pre></blockquote></div>
3503    *
3504    * @return A MasterCoprocessorRpcChannel instance
3505    */
3506   public CoprocessorRpcChannel coprocessorService() {
3507     return new MasterCoprocessorRpcChannel(connection);
3508   }
3509 
3510   /**
3511    * Creates and returns a {@link com.google.protobuf.RpcChannel} instance
3512    * connected to the passed region server.
3513    *
3514    * <p>
3515    * The obtained {@link com.google.protobuf.RpcChannel} instance can be used to access a published
3516    * coprocessor {@link com.google.protobuf.Service} using standard protobuf service invocations:
3517    * </p>
3518    *
3519    * <div style="background-color: #cccccc; padding: 2px">
3520    * <blockquote><pre>
3521    * CoprocessorRpcChannel channel = myAdmin.coprocessorService(serverName);
3522    * MyService.BlockingInterface service = MyService.newBlockingStub(channel);
3523    * MyCallRequest request = MyCallRequest.newBuilder()
3524    *     ...
3525    *     .build();
3526    * MyCallResponse response = service.myCall(null, request);
3527    * </pre></blockquote></div>
3528    *
3529    * @param sn the server name to which the endpoint call is made
3530    * @return A RegionServerCoprocessorRpcChannel instance
3531    */
3532   public CoprocessorRpcChannel coprocessorService(ServerName sn) {
3533     return new RegionServerCoprocessorRpcChannel(connection, sn);
3534   }
3535 
3536   /**
3537    * Truncate a table. Synchronous operation.
3538    * @param tableName name of table to truncate
3539    * @param preserveSplits True if the splits should be preserved
3540    * @throws IOException if a remote or network exception occurs
3541    */
3542   public void truncateTable(final TableName tableName, final boolean preserveSplits)
3543       throws IOException {
3544     executeCallable(new MasterCallable<Void>(getConnection()) {
3545       @Override
3546       public Void call() throws ServiceException {
3547         LOG.info("Started truncate of " + tableName);
3548         TruncateTableRequest req = RequestConverter.buildTruncateTableRequest(
3549         tableName, preserveSplits);
3550         master.truncateTable(null, req);
3551         return null;
3552       }
3553     });
3554   }
3555 
3556   /**
3557    * Return the set of supported security capabilities.
3558    * @throws IOException
3559    * @throws UnsupportedOperationException
3560    */
3561   public List<SecurityCapability> getSecurityCapabilities() throws IOException {
3562     try {
3563       return executeCallable(new MasterCallable<List<SecurityCapability>>(getConnection()) {
3564         @Override
3565         public List<SecurityCapability> call() throws ServiceException {
3566           SecurityCapabilitiesRequest req = SecurityCapabilitiesRequest.newBuilder().build();
3567           return ProtobufUtil.toSecurityCapabilityList(
3568             master.getSecurityCapabilities(null, req).getCapabilitiesList());
3569         }
3570       });
3571     } catch (IOException e) {
3572       if (e instanceof RemoteException) {
3573         e = ((RemoteException)e).unwrapRemoteException();
3574       }
3575       throw e;
3576     }
3577   }
3578 }