1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import org.apache.commons.logging.Log;
21 import org.apache.commons.logging.LogFactory;
22 import org.apache.hadoop.classification.InterfaceAudience;
23 import org.apache.hadoop.classification.InterfaceStability;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.HRegionInfo;
27 import org.apache.hadoop.hbase.KeyValue;
28 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
29 import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
30 import org.apache.hadoop.hbase.exceptions.NotServingRegionException;
31 import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
32 import org.apache.hadoop.hbase.exceptions.RegionServerStoppedException;
33 import org.apache.hadoop.hbase.exceptions.UnknownScannerException;
34 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
35 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
36 import org.apache.hadoop.hbase.util.Bytes;
37
38 import java.io.IOException;
39 import java.util.ArrayList;
40 import java.util.LinkedList;
41
42
43
44
45
46
47 @InterfaceAudience.Public
48 @InterfaceStability.Stable
49 public class ClientScanner extends AbstractClientScanner {
50 private final Log LOG = LogFactory.getLog(this.getClass());
51 private Scan scan;
52 private boolean closed = false;
53
54
55 private HRegionInfo currentRegion = null;
56 private ScannerCallable callable = null;
57 private final LinkedList<Result> cache = new LinkedList<Result>();
58 private final int caching;
59 private long lastNext;
60
61 private Result lastResult = null;
62 private ScanMetrics scanMetrics = null;
63 private final long maxScannerResultSize;
64 private final HConnection connection;
65 private final byte[] tableName;
66 private final int scannerTimeout;
67
68
69
70
71
72
73
74
75
76
77
78 public ClientScanner(final Configuration conf, final Scan scan,
79 final byte[] tableName) throws IOException {
80 this(conf, scan, tableName, HConnectionManager.getConnection(conf));
81 }
82
83
84
85
86
87
88
89
90
91
92
93 public ClientScanner(final Configuration conf, final Scan scan,
94 final byte[] tableName, HConnection connection) throws IOException {
95 if (LOG.isDebugEnabled()) {
96 LOG.debug("Creating scanner over "
97 + Bytes.toString(tableName)
98 + " starting at key '" + Bytes.toStringBinary(scan.getStartRow()) + "'");
99 }
100 this.scan = scan;
101 this.tableName = tableName;
102 this.lastNext = System.currentTimeMillis();
103 this.connection = connection;
104 if (scan.getMaxResultSize() > 0) {
105 this.maxScannerResultSize = scan.getMaxResultSize();
106 } else {
107 this.maxScannerResultSize = conf.getLong(
108 HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
109 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
110 }
111 this.scannerTimeout = conf.getInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
112 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
113
114
115 byte[] enableMetrics = scan.getAttribute(
116 Scan.SCAN_ATTRIBUTES_METRICS_ENABLE);
117 if (enableMetrics != null && Bytes.toBoolean(enableMetrics)) {
118 scanMetrics = new ScanMetrics();
119 }
120
121
122 if (this.scan.getCaching() > 0) {
123 this.caching = this.scan.getCaching();
124 } else {
125 this.caching = conf.getInt(
126 HConstants.HBASE_CLIENT_SCANNER_CACHING,
127 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
128 }
129
130
131 nextScanner(this.caching, false);
132 }
133
134 protected HConnection getConnection() {
135 return this.connection;
136 }
137
138 protected byte[] getTableName() {
139 return this.tableName;
140 }
141
142 protected Scan getScan() {
143 return scan;
144 }
145
146 protected long getTimestamp() {
147 return lastNext;
148 }
149
150
151 private boolean checkScanStopRow(final byte [] endKey) {
152 if (this.scan.getStopRow().length > 0) {
153
154 byte [] stopRow = scan.getStopRow();
155 int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
156 endKey, 0, endKey.length);
157 if (cmp <= 0) {
158
159
160 return true;
161 }
162 }
163 return false;
164 }
165
166
167
168
169
170
171
172
173
174
175 private boolean nextScanner(int nbRows, final boolean done)
176 throws IOException {
177
178 if (this.callable != null) {
179 this.callable.setClose();
180 callable.withRetries();
181 this.callable = null;
182 }
183
184
185 byte [] localStartKey;
186
187
188 if (this.currentRegion != null) {
189 byte [] endKey = this.currentRegion.getEndKey();
190 if (endKey == null ||
191 Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
192 checkScanStopRow(endKey) ||
193 done) {
194 close();
195 if (LOG.isDebugEnabled()) {
196 LOG.debug("Finished with scanning at " + this.currentRegion);
197 }
198 return false;
199 }
200 localStartKey = endKey;
201 if (LOG.isDebugEnabled()) {
202 LOG.debug("Finished with region " + this.currentRegion);
203 }
204 } else {
205 localStartKey = this.scan.getStartRow();
206 }
207
208 if (LOG.isDebugEnabled()) {
209 LOG.debug("Advancing internal scanner to startKey at '" +
210 Bytes.toStringBinary(localStartKey) + "'");
211 }
212 try {
213 callable = getScannerCallable(localStartKey, nbRows);
214
215
216 callable.withRetries();
217 this.currentRegion = callable.getHRegionInfo();
218 if (this.scanMetrics != null) {
219 this.scanMetrics.countOfRegions.incrementAndGet();
220 }
221 } catch (IOException e) {
222 close();
223 throw e;
224 }
225 return true;
226 }
227
228 protected ScannerCallable getScannerCallable(byte [] localStartKey,
229 int nbRows) {
230 scan.setStartRow(localStartKey);
231 ScannerCallable s = new ScannerCallable(getConnection(),
232 getTableName(), scan, this.scanMetrics);
233 s.setCaching(nbRows);
234 return s;
235 }
236
237
238
239
240
241
242
243
244
245
246
247
248 private void writeScanMetrics() throws IOException {
249 if (this.scanMetrics == null) {
250 return;
251 }
252 MapReduceProtos.ScanMetrics pScanMetrics = ProtobufUtil.toScanMetrics(scanMetrics);
253 scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, pScanMetrics.toByteArray());
254 }
255
256 public Result next() throws IOException {
257
258 if (cache.size() == 0 && this.closed) {
259 return null;
260 }
261 if (cache.size() == 0) {
262 Result [] values = null;
263 long remainingResultSize = maxScannerResultSize;
264 int countdown = this.caching;
265
266
267 callable.setCaching(this.caching);
268
269
270 boolean skipFirst = false;
271 boolean retryAfterOutOfOrderException = true;
272 do {
273 try {
274 if (skipFirst) {
275
276
277 callable.setCaching(1);
278 values = callable.withRetries();
279 callable.setCaching(this.caching);
280 skipFirst = false;
281 }
282
283
284
285 values = callable.withRetries();
286 retryAfterOutOfOrderException = true;
287 } catch (DoNotRetryIOException e) {
288 if (e instanceof UnknownScannerException) {
289 long timeout = lastNext + scannerTimeout;
290
291
292
293 if (timeout < System.currentTimeMillis()) {
294 long elapsed = System.currentTimeMillis() - lastNext;
295 ScannerTimeoutException ex = new ScannerTimeoutException(
296 elapsed + "ms passed since the last invocation, " +
297 "timeout is currently set to " + scannerTimeout);
298 ex.initCause(e);
299 throw ex;
300 }
301 } else {
302 Throwable cause = e.getCause();
303 if ((cause == null || (!(cause instanceof NotServingRegionException)
304 && !(cause instanceof RegionServerStoppedException)))
305 && !(e instanceof OutOfOrderScannerNextException)) {
306 throw e;
307 }
308 }
309
310
311 if (this.lastResult != null) {
312 this.scan.setStartRow(this.lastResult.getRow());
313
314
315 skipFirst = true;
316 }
317 if (e instanceof OutOfOrderScannerNextException) {
318 if (retryAfterOutOfOrderException) {
319 retryAfterOutOfOrderException = false;
320 } else {
321 throw new DoNotRetryIOException("Failed after retry"
322 + ", it could be cause by rpc timeout", e);
323 }
324 }
325
326 this.currentRegion = null;
327 callable = null;
328 continue;
329 }
330 long currentTime = System.currentTimeMillis();
331 if (this.scanMetrics != null ) {
332 this.scanMetrics.sumOfMillisSecBetweenNexts.addAndGet(currentTime-lastNext);
333 }
334 lastNext = currentTime;
335 if (values != null && values.length > 0) {
336 for (Result rs : values) {
337 cache.add(rs);
338 for (KeyValue kv : rs.raw()) {
339 remainingResultSize -= kv.heapSize();
340 }
341 countdown--;
342 this.lastResult = rs;
343 }
344 }
345
346 } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
347 }
348
349 if (cache.size() > 0) {
350 return cache.poll();
351 }
352
353
354 writeScanMetrics();
355 return null;
356 }
357
358
359
360
361
362
363
364
365
366
367 public Result [] next(int nbRows) throws IOException {
368
369 ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
370 for(int i = 0; i < nbRows; i++) {
371 Result next = next();
372 if (next != null) {
373 resultSets.add(next);
374 } else {
375 break;
376 }
377 }
378 return resultSets.toArray(new Result[resultSets.size()]);
379 }
380
381 public void close() {
382 if (callable != null) {
383 callable.setClose();
384 try {
385 callable.withRetries();
386 } catch (IOException e) {
387
388
389
390
391 } finally {
392
393 try {
394 writeScanMetrics();
395 } catch (IOException e) {
396
397 }
398 }
399 callable = null;
400 }
401 closed = true;
402 }
403 }