1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.catalog;
19
20 import com.google.common.annotations.VisibleForTesting;
21 import com.google.common.base.Stopwatch;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.hbase.Abortable;
28 import org.apache.hadoop.hbase.HRegionInfo;
29 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
30 import org.apache.hadoop.hbase.ServerName;
31 import org.apache.hadoop.hbase.client.HConnection;
32 import org.apache.hadoop.hbase.client.HConnectionManager;
33 import org.apache.hadoop.hbase.client.HTable;
34 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
35 import org.apache.hadoop.hbase.ipc.RpcClient.FailedServerException;
36 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
37 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
38 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
39 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
42 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
43 import org.apache.hadoop.ipc.RemoteException;
44
45 import java.io.EOFException;
46 import java.io.IOException;
47 import java.net.ConnectException;
48 import java.net.NoRouteToHostException;
49 import java.net.SocketException;
50 import java.net.SocketTimeoutException;
51 import java.net.UnknownHostException;
52
53
54
55
56
57
58
59
60
61
62
63
64 @InterfaceAudience.Private
65 public class CatalogTracker {
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111 private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
112 private final HConnection connection;
113 private final ZooKeeperWatcher zookeeper;
114 private final MetaRegionTracker metaRegionTracker;
115 private boolean instantiatedzkw = false;
116 private Abortable abortable;
117
118 private volatile boolean stopped = false;
119
120 static final byte [] META_REGION_NAME =
121 HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
122
123
124
125
126
127
128
129
130
131
132
133
134 public CatalogTracker(final Configuration conf) throws IOException {
135 this(null, conf, null);
136 }
137
138
139
140
141
142
143
144
145
146
147
148
149
150 public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
151 Abortable abortable)
152 throws IOException {
153 this(zk, conf, HConnectionManager.getConnection(conf), abortable);
154 }
155
156 public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
157 HConnection connection, Abortable abortable)
158 throws IOException {
159 this.connection = connection;
160 if (abortable == null) {
161
162 this.abortable = this.connection;
163 }
164 Abortable throwableAborter = new Abortable() {
165
166 @Override
167 public void abort(String why, Throwable e) {
168 throw new RuntimeException(why, e);
169 }
170
171 @Override
172 public boolean isAborted() {
173 return true;
174 }
175
176 };
177 if (zk == null) {
178
179 this.zookeeper =
180 new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(),
181 abortable);
182 instantiatedzkw = true;
183 } else {
184 this.zookeeper = zk;
185 }
186 this.metaRegionTracker = new MetaRegionTracker(zookeeper, throwableAborter);
187 }
188
189
190
191
192
193
194
195
196 public void start() throws IOException, InterruptedException {
197 LOG.debug("Starting catalog tracker " + this);
198 try {
199 this.metaRegionTracker.start();
200 } catch (RuntimeException e) {
201 Throwable t = e.getCause();
202 this.abortable.abort(e.getMessage(), t);
203 throw new IOException("Attempt to start meta tracker failed.", t);
204 }
205 }
206
207
208
209
210 @VisibleForTesting
211 public boolean isStopped() {
212 return this.stopped;
213 }
214
215
216
217
218
219 public void stop() {
220 if (!this.stopped) {
221 LOG.debug("Stopping catalog tracker " + this);
222 this.stopped = true;
223 this.metaRegionTracker.stop();
224 try {
225 if (this.connection != null) {
226 this.connection.close();
227 }
228 } catch (IOException e) {
229
230
231 LOG.error("Attempt to close catalog tracker's connection failed.", e);
232 }
233 if (this.instantiatedzkw) {
234 this.zookeeper.close();
235 }
236 }
237 }
238
239
240
241
242
243
244
245
246 public ServerName getMetaLocation() throws InterruptedException {
247 return this.metaRegionTracker.getMetaRegionLocation();
248 }
249
250
251
252
253
254 public boolean isMetaLocationAvailable() {
255 return this.metaRegionTracker.isLocationAvailable();
256 }
257
258
259
260
261
262
263
264
265
266
267
268 public ServerName waitForMeta(final long timeout)
269 throws InterruptedException, NotAllMetaRegionsOnlineException {
270 ServerName sn = metaRegionTracker.waitMetaRegionLocation(timeout);
271 if (sn == null) {
272 throw new NotAllMetaRegionsOnlineException("Timed out; " + timeout + "ms");
273 }
274 return sn;
275 }
276
277
278
279
280
281
282
283
284
285
286
287
288 public AdminService.BlockingInterface waitForMetaServerConnection(long timeout)
289 throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
290 return getMetaServerConnection(timeout);
291 }
292
293
294
295
296
297
298
299
300
301
302
303
304 AdminService.BlockingInterface getMetaServerConnection(long timeout)
305 throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
306 return getCachedConnection(waitForMeta(timeout));
307 }
308
309
310
311
312
313
314
315
316 public void waitForMeta() throws InterruptedException {
317 Stopwatch stopwatch = new Stopwatch().start();
318 while (!this.stopped) {
319 try {
320 if (waitForMeta(100) != null) break;
321 long sleepTime = stopwatch.elapsedMillis();
322
323 if ((sleepTime + 1) % 10000 == 0) {
324 LOG.warn("Have been waiting for meta to be assigned for " + sleepTime + "ms");
325 }
326 } catch (NotAllMetaRegionsOnlineException e) {
327 if (LOG.isTraceEnabled()) {
328 LOG.trace("hbase:meta still not available, sleeping and retrying." +
329 " Reason: " + e.getMessage());
330 }
331 }
332 }
333 }
334
335
336
337
338
339
340
341
342 private AdminService.BlockingInterface getCachedConnection(ServerName sn)
343 throws IOException {
344 if (sn == null) {
345 return null;
346 }
347 AdminService.BlockingInterface service = null;
348 try {
349 service = connection.getAdmin(sn);
350 } catch (RetriesExhaustedException e) {
351 if (e.getCause() != null && e.getCause() instanceof ConnectException) {
352
353 } else {
354 throw e;
355 }
356 } catch (SocketTimeoutException e) {
357 LOG.debug("Timed out connecting to " + sn);
358 } catch (NoRouteToHostException e) {
359 LOG.debug("Connecting to " + sn, e);
360 } catch (SocketException e) {
361 LOG.debug("Exception connecting to " + sn);
362 } catch (UnknownHostException e) {
363 LOG.debug("Unknown host exception connecting to " + sn);
364 } catch (FailedServerException e) {
365 if (LOG.isDebugEnabled()) {
366 LOG.debug("Server " + sn + " is in failed server list.");
367 }
368 } catch (IOException ioe) {
369 Throwable cause = ioe.getCause();
370 if (ioe instanceof ConnectException) {
371
372 } else if (cause != null && cause instanceof EOFException) {
373
374 } else if (cause != null && cause.getMessage() != null &&
375 cause.getMessage().toLowerCase().contains("connection reset")) {
376
377 } else {
378 throw ioe;
379 }
380
381 }
382 return service;
383 }
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400 private boolean verifyRegionLocation(AdminService.BlockingInterface hostingServer,
401 final ServerName address, final byte [] regionName)
402 throws IOException {
403 if (hostingServer == null) {
404 LOG.info("Passed hostingServer is null");
405 return false;
406 }
407 Throwable t = null;
408 try {
409
410 return ProtobufUtil.getRegionInfo(hostingServer, regionName) != null;
411 } catch (ConnectException e) {
412 t = e;
413 } catch (RetriesExhaustedException e) {
414 t = e;
415 } catch (RemoteException e) {
416 IOException ioe = e.unwrapRemoteException();
417 t = ioe;
418 } catch (IOException e) {
419 Throwable cause = e.getCause();
420 if (cause != null && cause instanceof EOFException) {
421 t = cause;
422 } else if (cause != null && cause.getMessage() != null
423 && cause.getMessage().contains("Connection reset")) {
424 t = cause;
425 } else {
426 t = e;
427 }
428 }
429 LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) +
430 " at address=" + address + ", exception=" + t);
431 return false;
432 }
433
434
435
436
437
438
439
440
441
442 public boolean verifyMetaRegionLocation(final long timeout)
443 throws InterruptedException, IOException {
444 AdminService.BlockingInterface service = null;
445 try {
446 service = waitForMetaServerConnection(timeout);
447 } catch (NotAllMetaRegionsOnlineException e) {
448
449 } catch (ServerNotRunningYetException e) {
450
451 } catch (UnknownHostException e) {
452
453 } catch (RegionServerStoppedException e) {
454
455 }
456 return (service == null)? false:
457 verifyRegionLocation(service,
458 this.metaRegionTracker.getMetaRegionLocation(), META_REGION_NAME);
459 }
460
461 public HConnection getConnection() {
462 return this.connection;
463 }
464 }