1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.client;
22
23
24 import com.google.protobuf.ServiceException;
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.DoNotRetryIOException;
29 import org.apache.hadoop.hbase.HBaseIOException;
30 import org.apache.hadoop.hbase.HRegionLocation;
31 import org.apache.hadoop.hbase.RegionLocations;
32 import org.apache.hadoop.hbase.ServerName;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.classification.InterfaceAudience;
35 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
36 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
37 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
38 import org.apache.hadoop.hbase.protobuf.RequestConverter;
39 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
40 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
41
42 import org.apache.htrace.Trace;
43
44 import java.io.IOException;
45 import java.io.InterruptedIOException;
46 import java.util.Collections;
47 import java.util.List;
48 import java.util.concurrent.CancellationException;
49 import java.util.concurrent.ExecutionException;
50 import java.util.concurrent.Executor;
51 import java.util.concurrent.ExecutorService;
52 import java.util.concurrent.Future;
53 import java.util.concurrent.RunnableFuture;
54 import java.util.concurrent.TimeUnit;
55 import java.util.concurrent.TimeoutException;
56
57
58
59
60
61
62
63 @InterfaceAudience.Private
64 public class RpcRetryingCallerWithReadReplicas {
65 static final Log LOG = LogFactory.getLog(RpcRetryingCallerWithReadReplicas.class);
66
67 protected final ExecutorService pool;
68 protected final ClusterConnection cConnection;
69 protected final Configuration conf;
70 protected final Get get;
71 protected final TableName tableName;
72 protected final int timeBeforeReplicas;
73 private final int callTimeout;
74 private final int retries;
75 private final RpcControllerFactory rpcControllerFactory;
76 private final RpcRetryingCallerFactory rpcRetryingCallerFactory;
77
78 public RpcRetryingCallerWithReadReplicas(
79 RpcControllerFactory rpcControllerFactory, TableName tableName,
80 ClusterConnection cConnection, final Get get,
81 ExecutorService pool, int retries, int callTimeout,
82 int timeBeforeReplicas) {
83 this.rpcControllerFactory = rpcControllerFactory;
84 this.tableName = tableName;
85 this.cConnection = cConnection;
86 this.conf = cConnection.getConfiguration();
87 this.get = get;
88 this.pool = pool;
89 this.retries = retries;
90 this.callTimeout = callTimeout;
91 this.timeBeforeReplicas = timeBeforeReplicas;
92 this.rpcRetryingCallerFactory = new RpcRetryingCallerFactory(conf);
93 }
94
95
96
97
98
99
100
101 class ReplicaRegionServerCallable extends RegionServerCallable<Result> {
102 final int id;
103 private final PayloadCarryingRpcController controller;
104
105 public ReplicaRegionServerCallable(int id, HRegionLocation location) {
106 super(RpcRetryingCallerWithReadReplicas.this.cConnection,
107 RpcRetryingCallerWithReadReplicas.this.tableName, get.getRow());
108 this.id = id;
109 this.location = location;
110 this.controller = rpcControllerFactory.newController();
111 controller.setPriority(tableName);
112 }
113
114 public void startCancel() {
115 controller.startCancel();
116 }
117
118
119
120
121
122
123 @Override
124 public void prepare(final boolean reload) throws IOException {
125 if (controller.isCanceled()) return;
126
127 if (Thread.interrupted()) {
128 throw new InterruptedIOException();
129 }
130
131 if (reload || location == null) {
132 RegionLocations rl = getRegionLocations(false, id, cConnection, tableName, get.getRow());
133 location = id < rl.size() ? rl.getRegionLocation(id) : null;
134 }
135
136 if (location == null || location.getServerName() == null) {
137
138
139 throw new HBaseIOException("There is no location for replica id #" + id);
140 }
141
142 ServerName dest = location.getServerName();
143
144 setStub(cConnection.getClient(dest));
145 }
146
147 @Override
148 public Result call(int callTimeout) throws Exception {
149 if (controller.isCanceled()) return null;
150
151 if (Thread.interrupted()) {
152 throw new InterruptedIOException();
153 }
154
155 byte[] reg = location.getRegionInfo().getRegionName();
156
157 ClientProtos.GetRequest request =
158 RequestConverter.buildGetRequest(reg, get);
159 controller.setCallTimeout(callTimeout);
160
161 try {
162 ClientProtos.GetResponse response = getStub().get(controller, request);
163 if (response == null) {
164 return null;
165 }
166 return ProtobufUtil.toResult(response.getResult());
167 } catch (ServiceException se) {
168 throw ProtobufUtil.getRemoteException(se);
169 }
170 }
171 }
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190 public synchronized Result call()
191 throws DoNotRetryIOException, InterruptedIOException, RetriesExhaustedException {
192 boolean isTargetReplicaSpecified = (get.getReplicaId() >= 0);
193
194 RegionLocations rl = getRegionLocations(true, (isTargetReplicaSpecified ? get.getReplicaId()
195 : RegionReplicaUtil.DEFAULT_REPLICA_ID), cConnection, tableName, get.getRow());
196 ResultBoundedCompletionService cs = new ResultBoundedCompletionService(pool, rl.size());
197
198 if(isTargetReplicaSpecified) {
199 addCallsForReplica(cs, rl, get.getReplicaId(), get.getReplicaId());
200 } else {
201 addCallsForReplica(cs, rl, 0, 0);
202 try {
203
204 Future<Result> f = cs.poll(timeBeforeReplicas, TimeUnit.MICROSECONDS);
205 if (f != null) {
206 return f.get();
207 }
208 } catch (ExecutionException e) {
209 throwEnrichedException(e, retries);
210 } catch (CancellationException e) {
211 throw new InterruptedIOException();
212 } catch (InterruptedException e) {
213 throw new InterruptedIOException();
214 }
215
216
217 addCallsForReplica(cs, rl, 1, rl.size() - 1);
218 }
219
220 try {
221 try {
222 Future<Result> f = cs.take();
223 return f.get();
224 } catch (ExecutionException e) {
225 throwEnrichedException(e, retries);
226 }
227 } catch (CancellationException e) {
228 throw new InterruptedIOException();
229 } catch (InterruptedException e) {
230 throw new InterruptedIOException();
231 } finally {
232
233
234 cs.cancelAll();
235 }
236
237 return null;
238 }
239
240
241
242
243
244 static void throwEnrichedException(ExecutionException e, int retries)
245 throws RetriesExhaustedException, DoNotRetryIOException {
246 Throwable t = e.getCause();
247 assert t != null;
248
249 if (t instanceof RetriesExhaustedException) {
250 throw (RetriesExhaustedException) t;
251 }
252
253 if (t instanceof DoNotRetryIOException) {
254 throw (DoNotRetryIOException) t;
255 }
256
257 RetriesExhaustedException.ThrowableWithExtraContext qt =
258 new RetriesExhaustedException.ThrowableWithExtraContext(t,
259 EnvironmentEdgeManager.currentTime(), null);
260
261 List<RetriesExhaustedException.ThrowableWithExtraContext> exceptions =
262 Collections.singletonList(qt);
263
264 throw new RetriesExhaustedException(retries, exceptions);
265 }
266
267
268
269
270
271
272
273
274
275 private void addCallsForReplica(ResultBoundedCompletionService cs,
276 RegionLocations rl, int min, int max) {
277 for (int id = min; id <= max; id++) {
278 HRegionLocation hrl = rl.getRegionLocation(id);
279 ReplicaRegionServerCallable callOnReplica = new ReplicaRegionServerCallable(id, hrl);
280 cs.submit(callOnReplica, callTimeout);
281 }
282 }
283
284 static RegionLocations getRegionLocations(boolean useCache, int replicaId,
285 ClusterConnection cConnection, TableName tableName, byte[] row)
286 throws RetriesExhaustedException, DoNotRetryIOException, InterruptedIOException {
287
288 RegionLocations rl;
289 try {
290 if (!useCache) {
291 rl = cConnection.relocateRegion(tableName, row, replicaId);
292 } else {
293 rl = cConnection.locateRegion(tableName, row, useCache, true, replicaId);
294 }
295 } catch (DoNotRetryIOException e) {
296 throw e;
297 } catch (RetriesExhaustedException e) {
298 throw e;
299 } catch (InterruptedIOException e) {
300 throw e;
301 } catch (IOException e) {
302 throw new RetriesExhaustedException("Can't get the location", e);
303 }
304 if (rl == null) {
305 throw new RetriesExhaustedException("Can't get the locations");
306 }
307
308 return rl;
309 }
310
311
312
313
314
315
316
317
318 public class ResultBoundedCompletionService {
319 private final Executor executor;
320 private final QueueingFuture[] tasks;
321 private volatile QueueingFuture completed = null;
322
323 class QueueingFuture implements RunnableFuture<Result> {
324 private final ReplicaRegionServerCallable future;
325 private Result result = null;
326 private ExecutionException exeEx = null;
327 private volatile boolean canceled;
328 private final int callTimeout;
329 private final RpcRetryingCaller<Result> retryingCaller;
330
331
332 public QueueingFuture(ReplicaRegionServerCallable future, int callTimeout) {
333 this.future = future;
334 this.callTimeout = callTimeout;
335 this.retryingCaller = rpcRetryingCallerFactory.<Result>newCaller();
336 }
337
338 @Override
339 public void run() {
340 try {
341 if (!canceled) {
342 result =
343 rpcRetryingCallerFactory.<Result>newCaller().callWithRetries(future, callTimeout);
344 }
345 } catch (Throwable t) {
346 exeEx = new ExecutionException(t);
347 } finally {
348 if (!canceled && completed == null) {
349 completed = QueueingFuture.this;
350 synchronized (tasks) {
351 tasks.notify();
352 }
353 }
354 }
355 }
356
357 @Override
358 public boolean cancel(boolean mayInterruptIfRunning) {
359 if (result != null || exeEx != null) return false;
360 retryingCaller.cancel();
361 future.startCancel();
362 canceled = true;
363 return true;
364 }
365
366 @Override
367 public boolean isCancelled() {
368 return canceled;
369 }
370
371 @Override
372 public boolean isDone() {
373 return result != null || exeEx != null;
374 }
375
376 @Override
377 public Result get() throws InterruptedException, ExecutionException {
378 try {
379 return get(1000, TimeUnit.DAYS);
380 } catch (TimeoutException e) {
381 throw new RuntimeException("You did wait for 1000 days here?", e);
382 }
383 }
384
385 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="RCN_REDUNDANT_NULLCHECK_OF_NULL_VALUE",
386 justification="Is this an issue?")
387 @Override
388 public Result get(long timeout, TimeUnit unit)
389 throws InterruptedException, ExecutionException, TimeoutException {
390 synchronized (tasks) {
391 if (result != null) {
392 return result;
393 }
394 if (exeEx != null) {
395 throw exeEx;
396 }
397 unit.timedWait(tasks, timeout);
398 }
399
400 if (result != null) {
401 return result;
402 }
403 if (exeEx != null) {
404 throw exeEx;
405 }
406
407 throw new TimeoutException("timeout=" + timeout + ", " + unit);
408 }
409 }
410
411 public ResultBoundedCompletionService(Executor executor, int maxTasks) {
412 this.executor = executor;
413 this.tasks = new QueueingFuture[maxTasks];
414 }
415
416
417 public void submit(ReplicaRegionServerCallable task, int callTimeout) {
418 QueueingFuture newFuture = new QueueingFuture(task, callTimeout);
419 executor.execute(Trace.wrap(newFuture));
420 tasks[task.id] = newFuture;
421 }
422
423 public QueueingFuture take() throws InterruptedException {
424 synchronized (tasks) {
425 while (completed == null) tasks.wait();
426 }
427 return completed;
428 }
429
430 public QueueingFuture poll(long timeout, TimeUnit unit) throws InterruptedException {
431 synchronized (tasks) {
432 if (completed == null) unit.timedWait(tasks, timeout);
433 }
434 return completed;
435 }
436
437 public void cancelAll() {
438 for (QueueingFuture future : tasks) {
439 if (future != null) future.cancel(true);
440 }
441 }
442 }
443 }