1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.lang.reflect.UndeclaredThrowableException;
25 import java.net.SocketTimeoutException;
26 import java.util.ArrayList;
27 import java.util.List;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.classification.InterfaceAudience;
32 import org.apache.hadoop.hbase.DoNotRetryIOException;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.ipc.RpcClient;
35 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
36 import org.apache.hadoop.hbase.util.ExceptionUtil;
37 import org.apache.hadoop.ipc.RemoteException;
38
39 import com.google.protobuf.ServiceException;
40
41
42
43
44
45
46 @InterfaceAudience.Private
47 @edu.umd.cs.findbugs.annotations.SuppressWarnings
48 (value = "IS2_INCONSISTENT_SYNC", justification = "na")
49 public class RpcRetryingCaller<T> {
50 static final Log LOG = LogFactory.getLog(RpcRetryingCaller.class);
51
52
53
54 private int callTimeout;
55
56
57
58 private long globalStartTime;
59
60
61
62 private final static int MIN_RPC_TIMEOUT = 2000;
63
64 private final int startLogErrorsCnt;
65
66 private final long pause;
67 private final int retries;
68
69 public RpcRetryingCaller(long pause, int retries, int startLogErrorsCnt) {
70 this.pause = pause;
71 this.retries = retries;
72 this.startLogErrorsCnt = startLogErrorsCnt;
73 }
74
75 private void beforeCall() {
76 int remaining = (int)(callTimeout -
77 (EnvironmentEdgeManager.currentTimeMillis() - this.globalStartTime));
78 if (remaining < MIN_RPC_TIMEOUT) {
79
80
81
82 remaining = MIN_RPC_TIMEOUT;
83 }
84 RpcClient.setRpcTimeout(remaining);
85 }
86
87 private void afterCall() {
88 RpcClient.resetRpcTimeout();
89 }
90
91 public synchronized T callWithRetries(RetryingCallable<T> callable) throws IOException,
92 RuntimeException {
93 return callWithRetries(callable, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
94 }
95
96
97
98
99
100
101
102
103
104 @edu.umd.cs.findbugs.annotations.SuppressWarnings
105 (value = "SWL_SLEEP_WITH_LOCK_HELD", justification = "na")
106 public synchronized T callWithRetries(RetryingCallable<T> callable, int callTimeout)
107 throws IOException, RuntimeException {
108 this.callTimeout = callTimeout;
109 List<RetriesExhaustedException.ThrowableWithExtraContext> exceptions =
110 new ArrayList<RetriesExhaustedException.ThrowableWithExtraContext>();
111 this.globalStartTime = EnvironmentEdgeManager.currentTimeMillis();
112 for (int tries = 0;; tries++) {
113 long expectedSleep = 0;
114 try {
115 beforeCall();
116 callable.prepare(tries != 0);
117 return callable.call();
118 } catch (Throwable t) {
119 if (tries > startLogErrorsCnt) {
120 LOG.info("Call exception, tries=" + tries + ", retries=" + retries + ", retryTime=" +
121 (EnvironmentEdgeManager.currentTimeMillis() - this.globalStartTime) + "ms, msg="
122 + callable.getExceptionMessageAdditionalDetail());
123 }
124
125 t = translateException(t);
126 callable.throwable(t, retries != 1);
127 RetriesExhaustedException.ThrowableWithExtraContext qt =
128 new RetriesExhaustedException.ThrowableWithExtraContext(t,
129 EnvironmentEdgeManager.currentTimeMillis(), toString());
130 exceptions.add(qt);
131 ExceptionUtil.rethrowIfInterrupt(t);
132 if (tries >= retries - 1) {
133 throw new RetriesExhaustedException(tries, exceptions);
134 }
135
136
137
138 expectedSleep = callable.sleep(pause, tries + 1);
139
140
141 long duration = singleCallDuration(expectedSleep);
142 if (duration > this.callTimeout) {
143 String msg = "callTimeout=" + this.callTimeout + ", callDuration=" + duration +
144 ": " + callable.getExceptionMessageAdditionalDetail();
145 throw (SocketTimeoutException)(new SocketTimeoutException(msg).initCause(t));
146 }
147 } finally {
148 afterCall();
149 }
150 try {
151 Thread.sleep(expectedSleep);
152 } catch (InterruptedException e) {
153 throw new InterruptedIOException("Interrupted after " + tries + " tries on " + retries);
154 }
155 }
156 }
157
158
159
160
161
162 private long singleCallDuration(final long expectedSleep) {
163 return (EnvironmentEdgeManager.currentTimeMillis() - this.globalStartTime)
164 + MIN_RPC_TIMEOUT + expectedSleep;
165 }
166
167
168
169
170
171
172
173
174
175
176 public T callWithoutRetries(RetryingCallable<T> callable, int callTimeout)
177 throws IOException, RuntimeException {
178
179 this.globalStartTime = EnvironmentEdgeManager.currentTimeMillis();
180 this.callTimeout = callTimeout;
181 try {
182 beforeCall();
183 callable.prepare(false);
184 return callable.call();
185 } catch (Throwable t) {
186 Throwable t2 = translateException(t);
187 ExceptionUtil.rethrowIfInterrupt(t2);
188
189 if (t2 instanceof IOException) {
190 throw (IOException)t2;
191 } else {
192 throw new RuntimeException(t2);
193 }
194 } finally {
195 afterCall();
196 }
197 }
198
199
200
201
202
203
204
205
206 static Throwable translateException(Throwable t) throws DoNotRetryIOException {
207 if (t instanceof UndeclaredThrowableException) {
208 if (t.getCause() != null) {
209 t = t.getCause();
210 }
211 }
212 if (t instanceof RemoteException) {
213 t = ((RemoteException)t).unwrapRemoteException();
214 }
215 if (t instanceof LinkageError) {
216 throw new DoNotRetryIOException(t);
217 }
218 if (t instanceof ServiceException) {
219 ServiceException se = (ServiceException)t;
220 Throwable cause = se.getCause();
221 if (cause != null && cause instanceof DoNotRetryIOException) {
222 throw (DoNotRetryIOException)cause;
223 }
224
225 t = cause;
226
227 translateException(t);
228 } else if (t instanceof DoNotRetryIOException) {
229 throw (DoNotRetryIOException)t;
230 }
231 return t;
232 }
233 }