View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import com.google.common.annotations.VisibleForTesting;
21  import com.google.common.base.Stopwatch;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.hbase.classification.InterfaceAudience;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.Abortable;
28  import org.apache.hadoop.hbase.HRegionInfo;
29  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
30  import org.apache.hadoop.hbase.ServerName;
31  import org.apache.hadoop.hbase.client.HConnection;
32  import org.apache.hadoop.hbase.client.HConnectionManager;
33  import org.apache.hadoop.hbase.client.HTable;
34  import org.apache.hadoop.hbase.client.RetriesExhaustedException;
35  import org.apache.hadoop.hbase.ipc.RpcClient.FailedServerException;
36  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
37  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
38  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
39  import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
42  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
43  import org.apache.hadoop.ipc.RemoteException;
44  
45  import java.io.EOFException;
46  import java.io.IOException;
47  import java.net.ConnectException;
48  import java.net.NoRouteToHostException;
49  import java.net.SocketException;
50  import java.net.SocketTimeoutException;
51  import java.net.UnknownHostException;
52  
53  /**
54   * Tracks the availability of the catalog tables
55   * <code>hbase:meta</code>.
56   *
57   * This class is "read-only" in that the locations of the catalog tables cannot
58   * be explicitly set.  Instead, ZooKeeper is used to learn of the availability
59   * and location of <code>hbase:meta</code>.
60   *
61   * <p>Call {@link #start()} to start up operation.  Call {@link #stop()}} to
62   * interrupt waits and close up shop.
63   */
64  @InterfaceAudience.Private
65  public class CatalogTracker {
66    // TODO JDC 11/30 We don't even have ROOT anymore, revisit
67    // TODO: This class needs a rethink.  The original intent was that it would be
68    // the one-stop-shop for meta locations and that it would get this
69    // info from reading and watching zk state.  The class was to be used by
70    // servers when they needed to know of meta movement but also by
71    // client-side (inside in HTable) so rather than figure meta
72    // locations on fault, the client would instead get notifications out of zk.
73    //
74    // But this original intent is frustrated by the fact that this class has to
75    // read an hbase table, the -ROOT- table, to figure out the hbase:meta region
76    // location which means we depend on an HConnection.  HConnection will do
77    // retrying but also, it has its own mechanism for finding root and meta
78    // locations (and for 'verifying'; it tries the location and if it fails, does
79    // new lookup, etc.).  So, at least for now, HConnection (or HTable) can't
80    // have a CT since CT needs a HConnection (Even then, do want HT to have a CT?
81    // For HT keep up a session with ZK?  Rather, shouldn't we do like asynchbase
82    // where we'd open a connection to zk, read what we need then let the
83    // connection go?).  The 'fix' is make it so both root and meta addresses
84    // are wholey up in zk -- not in zk (root) -- and in an hbase table (meta).
85    //
86    // But even then, this class does 'verification' of the location and it does
87    // this by making a call over an HConnection (which will do its own root
88    // and meta lookups).  Isn't this verification 'useless' since when we
89    // return, whatever is dependent on the result of this call then needs to
90    // use HConnection; what we have verified may change in meantime (HConnection
91    // uses the CT primitives, the root and meta trackers finding root locations).
92    //
93    // When meta is moved to zk, this class may make more sense.  In the
94    // meantime, it does not cohere.  It should just watch meta and root and not
95    // NOT do verification -- let that be out in HConnection since its going to
96    // be done there ultimately anyways.
97    //
98    // This class has spread throughout the codebase.  It needs to be reigned in.
99    // This class should be used server-side only, even if we move meta location
100   // up into zk.  Currently its used over in the client package. Its used in
101   // MetaReader and MetaEditor classes usually just to get the Configuration
102   // its using (It does this indirectly by asking its HConnection for its
103   // Configuration and even then this is just used to get an HConnection out on
104   // the other end). I made https://issues.apache.org/jira/browse/HBASE-4495 for
105   // doing CT fixup. St.Ack 09/30/2011.
106   //
107 
108   // TODO: Timeouts have never been as advertised in here and its worse now
109   // with retries; i.e. the HConnection retries and pause goes ahead whatever
110   // the passed timeout is.  Fix.
111   private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
112   private final HConnection connection;
113   private final ZooKeeperWatcher zookeeper;
114   private final MetaRegionTracker metaRegionTracker;
115   private boolean instantiatedzkw = false;
116   private Abortable abortable;
117 
118   private volatile boolean stopped = false;
119 
120   static final byte [] META_REGION_NAME =
121     HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
122 
123   /**
124    * Constructs a catalog tracker. Find current state of catalog tables.
125    * Begin active tracking by executing {@link #start()} post construction. Does
126    * not timeout.
127    *
128    * @param conf
129    *          the {@link Configuration} from which a {@link HConnection} will be
130    *          obtained; if problem, this connections
131    *          {@link HConnection#abort(String, Throwable)} will be called.
132    * @throws IOException
133    */
134   public CatalogTracker(final Configuration conf) throws IOException {
135     this(null, conf, null);
136   }
137 
138   /**
139    * Constructs the catalog tracker.  Find current state of catalog tables.
140    * Begin active tracking by executing {@link #start()} post construction.
141    * Does not timeout.
142    * @param zk If zk is null, we'll create an instance (and shut it down
143    * when {@link #stop()} is called) else we'll use what is passed.
144    * @param conf
145    * @param abortable If fatal exception we'll call abort on this.  May be null.
146    * If it is we'll use the Connection associated with the passed
147    * {@link Configuration} as our Abortable.
148    * @throws IOException
149    */
150   public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
151       Abortable abortable)
152   throws IOException {
153     this(zk, conf, HConnectionManager.getConnection(conf), abortable);
154   }
155 
156   public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
157       HConnection connection, Abortable abortable)
158   throws IOException {
159     this.connection = connection;
160     if (abortable == null) {
161       // A connection is abortable.
162       this.abortable = this.connection;
163     }
164     Abortable throwableAborter = new Abortable() {
165 
166       @Override
167       public void abort(String why, Throwable e) {
168         throw new RuntimeException(why, e);
169       }
170 
171       @Override
172       public boolean isAborted() {
173         return true;
174       }
175 
176     };
177     if (zk == null) {
178       // Create our own.  Set flag so we tear it down on stop.
179       this.zookeeper =
180         new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(),
181           abortable);
182       instantiatedzkw = true;
183     } else {
184       this.zookeeper = zk;
185     }
186     this.metaRegionTracker = new MetaRegionTracker(zookeeper, throwableAborter);
187   }
188 
189   /**
190    * Starts the catalog tracker.
191    * Determines current availability of catalog tables and ensures all further
192    * transitions of either region are tracked.
193    * @throws IOException
194    * @throws InterruptedException
195    */
196   public void start() throws IOException, InterruptedException {
197     LOG.debug("Starting catalog tracker " + this);
198     try {
199       this.metaRegionTracker.start();
200     } catch (RuntimeException e) {
201       Throwable t = e.getCause();
202       this.abortable.abort(e.getMessage(), t);
203       throw new IOException("Attempt to start meta tracker failed.", t);
204     }
205   }
206 
207   /**
208    * @return True if we are stopped. Call only after start else indeterminate answer.
209    */
210   @VisibleForTesting
211   public boolean isStopped() {
212     return this.stopped;
213   }
214 
215   /**
216    * Stop working.
217    * Interrupts any ongoing waits.
218    */
219   public void stop() {
220     if (!this.stopped) {
221       LOG.debug("Stopping catalog tracker " + this);
222       this.stopped = true;
223       this.metaRegionTracker.stop();
224       try {
225         if (this.connection != null) {
226           this.connection.close();
227         }
228       } catch (IOException e) {
229         // Although the {@link Closeable} interface throws an {@link
230         // IOException}, in reality, the implementation would never do that.
231         LOG.error("Attempt to close catalog tracker's connection failed.", e);
232       }
233       if (this.instantiatedzkw) {
234         this.zookeeper.close();
235       }
236     }
237   }
238 
239   /**
240    * Gets the current location for <code>hbase:meta</code> or null if location is
241    * not currently available.
242    * @return {@link ServerName} for server hosting <code>hbase:meta</code> or null
243    * if none available
244    * @throws InterruptedException
245    */
246   public ServerName getMetaLocation() throws InterruptedException {
247     return this.metaRegionTracker.getMetaRegionLocation();
248   }
249 
250   /**
251    * Checks whether meta regionserver znode has some non null data.
252    * @return true if data is not null, false otherwise.
253    */
254   public boolean isMetaLocationAvailable() {
255     return this.metaRegionTracker.isLocationAvailable();
256   }
257   /**
258    * Gets the current location for <code>hbase:meta</code> if available and waits
259    * for up to the specified timeout if not immediately available.  Returns null
260    * if the timeout elapses before root is available.
261    * @param timeout maximum time to wait for root availability, in milliseconds
262    * @return {@link ServerName} for server hosting <code>hbase:meta</code> or null
263    * if none available
264    * @throws InterruptedException if interrupted while waiting
265    * @throws NotAllMetaRegionsOnlineException if meta not available before
266    * timeout
267    */
268   public ServerName waitForMeta(final long timeout)
269   throws InterruptedException, NotAllMetaRegionsOnlineException {
270     ServerName sn = metaRegionTracker.waitMetaRegionLocation(timeout);
271     if (sn == null) {
272       throw new NotAllMetaRegionsOnlineException("Timed out; " + timeout + "ms");
273     }
274     return sn;
275   }
276 
277   /**
278    * Gets a connection to the server hosting meta, as reported by ZooKeeper,
279    * waiting up to the specified timeout for availability.
280    * @param timeout How long to wait on meta location
281    * @see #waitForMeta for additional information
282    * @return connection to server hosting meta
283    * @throws InterruptedException
284    * @throws NotAllMetaRegionsOnlineException if timed out waiting
285    * @throws IOException
286    * @deprecated Use #getMetaServerConnection(long)
287    */
288   public AdminService.BlockingInterface waitForMetaServerConnection(long timeout)
289   throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
290     return getMetaServerConnection(timeout);
291   }
292 
293   /**
294    * Gets a connection to the server hosting meta, as reported by ZooKeeper,
295    * waiting up to the specified timeout for availability.
296    * <p>WARNING: Does not retry.  Use an {@link HTable} instead.
297    * @param timeout How long to wait on meta location
298    * @see #waitForMeta for additional information
299    * @return connection to server hosting meta
300    * @throws InterruptedException
301    * @throws NotAllMetaRegionsOnlineException if timed out waiting
302    * @throws IOException
303    */
304   AdminService.BlockingInterface getMetaServerConnection(long timeout)
305   throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
306     return getCachedConnection(waitForMeta(timeout));
307   }
308 
309   /**
310    * Waits indefinitely for availability of <code>hbase:meta</code>.  Used during
311    * cluster startup.  Does not verify meta, just that something has been
312    * set up in zk.
313    * @see #waitForMeta(long)
314    * @throws InterruptedException if interrupted while waiting
315    */
316   public void waitForMeta() throws InterruptedException {
317     Stopwatch stopwatch = new Stopwatch().start();
318     while (!this.stopped) {
319       try {
320         if (waitForMeta(100) != null) break;
321         long sleepTime = stopwatch.elapsedMillis();
322         // +1 in case sleepTime=0
323         if ((sleepTime + 1) % 10000 == 0) {
324           LOG.warn("Have been waiting for meta to be assigned for " + sleepTime + "ms");
325         }
326       } catch (NotAllMetaRegionsOnlineException e) {
327         if (LOG.isTraceEnabled()) {
328           LOG.trace("hbase:meta still not available, sleeping and retrying." +
329           " Reason: " + e.getMessage());
330         }
331       }
332     }
333   }
334 
335   /**
336    * @param sn ServerName to get a connection against.
337    * @return The AdminProtocol we got when we connected to <code>sn</code>
338    * May have come from cache, may not be good, may have been setup by this
339    * invocation, or may be null.
340    * @throws IOException
341    */
342   private AdminService.BlockingInterface getCachedConnection(ServerName sn)
343   throws IOException {
344     if (sn == null) {
345       return null;
346     }
347     AdminService.BlockingInterface service = null;
348     try {
349       service = connection.getAdmin(sn);
350     } catch (RetriesExhaustedException e) {
351       if (e.getCause() != null && e.getCause() instanceof ConnectException) {
352         // Catch this; presume it means the cached connection has gone bad.
353       } else {
354         throw e;
355       }
356     } catch (SocketTimeoutException e) {
357       LOG.debug("Timed out connecting to " + sn);
358     } catch (NoRouteToHostException e) {
359       LOG.debug("Connecting to " + sn, e);
360     } catch (SocketException e) {
361       LOG.debug("Exception connecting to " + sn);
362     } catch (UnknownHostException e) {
363       LOG.debug("Unknown host exception connecting to  " + sn);
364     } catch (FailedServerException e) {
365       if (LOG.isDebugEnabled()) {
366         LOG.debug("Server " + sn + " is in failed server list.");
367       }
368     } catch (IOException ioe) {
369       Throwable cause = ioe.getCause();
370       if (ioe instanceof ConnectException) {
371         // Catch. Connect refused.
372       } else if (cause != null && cause instanceof EOFException) {
373         // Catch. Other end disconnected us.
374       } else if (cause != null && cause.getMessage() != null &&
375         cause.getMessage().toLowerCase().contains("connection reset")) {
376         // Catch. Connection reset.
377       } else {
378         throw ioe;
379       }
380 
381     }
382     return service;
383   }
384 
385   /**
386    * Verify we can connect to <code>hostingServer</code> and that its carrying
387    * <code>regionName</code>.
388    * @param hostingServer Interface to the server hosting <code>regionName</code>
389    * @param address The servername that goes with the <code>metaServer</code>
390    * Interface.  Used logging.
391    * @param regionName The regionname we are interested in.
392    * @return True if we were able to verify the region located at other side of
393    * the Interface.
394    * @throws IOException
395    */
396   // TODO: We should be able to get the ServerName from the AdminProtocol
397   // rather than have to pass it in.  Its made awkward by the fact that the
398   // HRI is likely a proxy against remote server so the getServerName needs
399   // to be fixed to go to a local method or to a cache before we can do this.
400   private boolean verifyRegionLocation(AdminService.BlockingInterface hostingServer,
401       final ServerName address, final byte [] regionName)
402   throws IOException {
403     if (hostingServer == null) {
404       LOG.info("Passed hostingServer is null");
405       return false;
406     }
407     Throwable t = null;
408     try {
409       // Try and get regioninfo from the hosting server.
410       return ProtobufUtil.getRegionInfo(hostingServer, regionName) != null;
411     } catch (ConnectException e) {
412       t = e;
413     } catch (RetriesExhaustedException e) {
414       t = e;
415     } catch (RemoteException e) {
416       IOException ioe = e.unwrapRemoteException();
417       t = ioe;
418     } catch (IOException e) {
419       Throwable cause = e.getCause();
420       if (cause != null && cause instanceof EOFException) {
421         t = cause;
422       } else if (cause != null && cause.getMessage() != null
423           && cause.getMessage().contains("Connection reset")) {
424         t = cause;
425       } else {
426         t = e;
427       }
428     }
429     LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) +
430       " at address=" + address + ", exception=" + t);
431     return false;
432   }
433 
434   /**
435    * Verify <code>hbase:meta</code> is deployed and accessible.
436    * @param timeout How long to wait on zk for meta address (passed through to
437    * the internal call to {@link #waitForMetaServerConnection(long)}.
438    * @return True if the <code>hbase:meta</code> location is healthy.
439    * @throws IOException
440    * @throws InterruptedException
441    */
442   public boolean verifyMetaRegionLocation(final long timeout)
443   throws InterruptedException, IOException {
444     AdminService.BlockingInterface service = null;
445     try {
446       service = waitForMetaServerConnection(timeout);
447     } catch (NotAllMetaRegionsOnlineException e) {
448       // Pass
449     } catch (ServerNotRunningYetException e) {
450       // Pass -- remote server is not up so can't be carrying root
451     } catch (UnknownHostException e) {
452       // Pass -- server name doesn't resolve so it can't be assigned anything.
453     } catch (RegionServerStoppedException e) {
454       // Pass -- server name sends us to a server that is dying or already dead.
455     }
456     return (service == null)? false:
457       verifyRegionLocation(service,
458           this.metaRegionTracker.getMetaRegionLocation(), META_REGION_NAME);
459   }
460 
461   public HConnection getConnection() {
462     return this.connection;
463   }
464 }