1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.LinkedList;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.classification.InterfaceAudience;
27 import org.apache.hadoop.classification.InterfaceStability;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.Cell;
30 import org.apache.hadoop.hbase.DoNotRetryIOException;
31 import org.apache.hadoop.hbase.HBaseConfiguration;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.KeyValueUtil;
35 import org.apache.hadoop.hbase.NotServingRegionException;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.UnknownScannerException;
38 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
39 import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
40 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
42 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
43 import org.apache.hadoop.hbase.util.Bytes;
44
45
46
47
48
49
50 @InterfaceAudience.Public
51 @InterfaceStability.Stable
52 public class ClientScanner extends AbstractClientScanner {
53 private final Log LOG = LogFactory.getLog(this.getClass());
54 protected Scan scan;
55 protected boolean closed = false;
56
57
58 protected HRegionInfo currentRegion = null;
59 private ScannerCallable callable = null;
60 protected final LinkedList<Result> cache = new LinkedList<Result>();
61 protected final int caching;
62 protected long lastNext;
63
64 protected Result lastResult = null;
65 protected ScanMetrics scanMetrics = null;
66 protected final long maxScannerResultSize;
67 private final HConnection connection;
68 private final TableName tableName;
69 private final int scannerTimeout;
70 protected boolean scanMetricsPublished = false;
71 protected RpcRetryingCaller<Result []> caller;
72
73
74
75
76
77
78
79
80
81
82
83 public ClientScanner(final Configuration conf, final Scan scan,
84 final TableName tableName) throws IOException {
85 this(conf, scan, tableName, HConnectionManager.getConnection(conf));
86 }
87
88
89
90
91 @Deprecated
92 public ClientScanner(final Configuration conf, final Scan scan,
93 final byte [] tableName) throws IOException {
94 this(conf, scan, TableName.valueOf(tableName));
95 }
96
97
98
99
100
101
102
103
104
105
106
107
108 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
109 HConnection connection) throws IOException {
110 this(conf, scan, tableName, connection, new RpcRetryingCallerFactory(conf));
111 }
112
113
114
115
116 @Deprecated
117 public ClientScanner(final Configuration conf, final Scan scan, final byte [] tableName,
118 HConnection connection) throws IOException {
119 this(conf, scan, TableName.valueOf(tableName), connection, new RpcRetryingCallerFactory(conf));
120 }
121
122
123
124
125
126
127
128
129
130
131 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
132 HConnection connection, RpcRetryingCallerFactory rpcFactory) throws IOException {
133 if (LOG.isTraceEnabled()) {
134 LOG.trace("Scan table=" + tableName
135 + ", startRow=" + Bytes.toStringBinary(scan.getStartRow()));
136 }
137 this.scan = scan;
138 this.tableName = tableName;
139 this.lastNext = System.currentTimeMillis();
140 this.connection = connection;
141 if (scan.getMaxResultSize() > 0) {
142 this.maxScannerResultSize = scan.getMaxResultSize();
143 } else {
144 this.maxScannerResultSize = conf.getLong(
145 HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
146 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
147 }
148 this.scannerTimeout = HBaseConfiguration.getInt(conf,
149 HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
150 HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
151 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
152
153
154 byte[] enableMetrics = scan.getAttribute(
155 Scan.SCAN_ATTRIBUTES_METRICS_ENABLE);
156 if (enableMetrics != null && Bytes.toBoolean(enableMetrics)) {
157 scanMetrics = new ScanMetrics();
158 }
159
160
161 if (this.scan.getCaching() > 0) {
162 this.caching = this.scan.getCaching();
163 } else {
164 this.caching = conf.getInt(
165 HConstants.HBASE_CLIENT_SCANNER_CACHING,
166 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
167 }
168
169 this.caller = rpcFactory.<Result[]> newCaller();
170
171 initializeScannerInConstruction();
172 }
173
174 protected void initializeScannerInConstruction() throws IOException{
175
176 nextScanner(this.caching, false);
177 }
178
179 protected HConnection getConnection() {
180 return this.connection;
181 }
182
183
184
185
186
187 @Deprecated
188 protected byte [] getTableName() {
189 return this.tableName.getName();
190 }
191
192 protected TableName getTable() {
193 return this.tableName;
194 }
195
196 protected Scan getScan() {
197 return scan;
198 }
199
200 protected long getTimestamp() {
201 return lastNext;
202 }
203
204
205 protected boolean checkScanStopRow(final byte [] endKey) {
206 if (this.scan.getStopRow().length > 0) {
207
208 byte [] stopRow = scan.getStopRow();
209 int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
210 endKey, 0, endKey.length);
211 if (cmp <= 0) {
212
213
214 return true;
215 }
216 }
217 return false;
218 }
219
220
221
222
223
224
225
226
227
228
229 private boolean nextScanner(int nbRows, final boolean done)
230 throws IOException {
231
232 if (this.callable != null) {
233 this.callable.setClose();
234 this.caller.callWithRetries(callable);
235 this.callable = null;
236 }
237
238
239 byte [] localStartKey;
240
241
242 if (this.currentRegion != null) {
243 byte [] endKey = this.currentRegion.getEndKey();
244 if (endKey == null ||
245 Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
246 checkScanStopRow(endKey) ||
247 done) {
248 close();
249 if (LOG.isTraceEnabled()) {
250 LOG.trace("Finished " + this.currentRegion);
251 }
252 return false;
253 }
254 localStartKey = endKey;
255 if (LOG.isTraceEnabled()) {
256 LOG.trace("Finished " + this.currentRegion);
257 }
258 } else {
259 localStartKey = this.scan.getStartRow();
260 }
261
262 if (LOG.isDebugEnabled() && this.currentRegion != null) {
263
264 LOG.debug("Advancing internal scanner to startKey at '" +
265 Bytes.toStringBinary(localStartKey) + "'");
266 }
267 try {
268 callable = getScannerCallable(localStartKey, nbRows);
269
270
271 this.caller.callWithRetries(callable);
272 this.currentRegion = callable.getHRegionInfo();
273 if (this.scanMetrics != null) {
274 this.scanMetrics.countOfRegions.incrementAndGet();
275 }
276 } catch (IOException e) {
277 close();
278 throw e;
279 }
280 return true;
281 }
282
283 @InterfaceAudience.Private
284 protected ScannerCallable getScannerCallable(byte [] localStartKey,
285 int nbRows) {
286 scan.setStartRow(localStartKey);
287 ScannerCallable s = new ScannerCallable(getConnection(),
288 getTable(), scan, this.scanMetrics);
289 s.setCaching(nbRows);
290 return s;
291 }
292
293
294
295
296
297
298
299
300
301
302
303
304 protected void writeScanMetrics() {
305 if (this.scanMetrics == null || scanMetricsPublished) {
306 return;
307 }
308 MapReduceProtos.ScanMetrics pScanMetrics = ProtobufUtil.toScanMetrics(scanMetrics);
309 scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, pScanMetrics.toByteArray());
310 scanMetricsPublished = true;
311 }
312
313 @Override
314 public Result next() throws IOException {
315
316 if (cache.size() == 0 && this.closed) {
317 return null;
318 }
319 if (cache.size() == 0) {
320 Result [] values = null;
321 long remainingResultSize = maxScannerResultSize;
322 int countdown = this.caching;
323
324
325 callable.setCaching(this.caching);
326
327
328 boolean skipFirst = false;
329 boolean retryAfterOutOfOrderException = true;
330 do {
331 try {
332 if (skipFirst) {
333
334
335 callable.setCaching(1);
336 values = this.caller.callWithRetries(callable);
337 callable.setCaching(this.caching);
338 skipFirst = false;
339 }
340
341
342
343 values = this.caller.callWithRetries(callable);
344 if (skipFirst && values != null && values.length == 1) {
345 skipFirst = false;
346 values = this.caller.callWithRetries(callable);
347 }
348 retryAfterOutOfOrderException = true;
349 } catch (DoNotRetryIOException e) {
350
351
352 if (e instanceof UnknownScannerException) {
353 long timeout = lastNext + scannerTimeout;
354
355
356
357 if (timeout < System.currentTimeMillis()) {
358 long elapsed = System.currentTimeMillis() - lastNext;
359 ScannerTimeoutException ex = new ScannerTimeoutException(
360 elapsed + "ms passed since the last invocation, " +
361 "timeout is currently set to " + scannerTimeout);
362 ex.initCause(e);
363 throw ex;
364 }
365 } else {
366
367
368 Throwable cause = e.getCause();
369 if ((cause != null && cause instanceof NotServingRegionException) ||
370 (cause != null && cause instanceof RegionServerStoppedException) ||
371 e instanceof OutOfOrderScannerNextException) {
372
373
374
375 } else {
376 throw e;
377 }
378 }
379
380 if (this.lastResult != null) {
381 this.scan.setStartRow(this.lastResult.getRow());
382
383
384 skipFirst = true;
385 }
386 if (e instanceof OutOfOrderScannerNextException) {
387 if (retryAfterOutOfOrderException) {
388 retryAfterOutOfOrderException = false;
389 } else {
390
391 throw new DoNotRetryIOException("Failed after retry of " +
392 "OutOfOrderScannerNextException: was there a rpc timeout?", e);
393 }
394 }
395
396 this.currentRegion = null;
397
398
399 callable = null;
400
401 continue;
402 }
403 long currentTime = System.currentTimeMillis();
404 if (this.scanMetrics != null ) {
405 this.scanMetrics.sumOfMillisSecBetweenNexts.addAndGet(currentTime-lastNext);
406 }
407 lastNext = currentTime;
408 if (values != null && values.length > 0) {
409 for (Result rs : values) {
410 cache.add(rs);
411 for (Cell kv : rs.rawCells()) {
412
413 remainingResultSize -= KeyValueUtil.ensureKeyValue(kv).heapSize();
414 }
415 countdown--;
416 this.lastResult = rs;
417 }
418 }
419
420 } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
421 }
422
423 if (cache.size() > 0) {
424 return cache.poll();
425 }
426
427
428 writeScanMetrics();
429 return null;
430 }
431
432
433
434
435
436
437
438
439
440
441 @Override
442 public Result [] next(int nbRows) throws IOException {
443
444 ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
445 for(int i = 0; i < nbRows; i++) {
446 Result next = next();
447 if (next != null) {
448 resultSets.add(next);
449 } else {
450 break;
451 }
452 }
453 return resultSets.toArray(new Result[resultSets.size()]);
454 }
455
456 @Override
457 public void close() {
458 if (!scanMetricsPublished) writeScanMetrics();
459 if (callable != null) {
460 callable.setClose();
461 try {
462 this.caller.callWithRetries(callable);
463 } catch (IOException e) {
464
465
466
467
468 }
469 callable = null;
470 }
471 closed = true;
472 }
473 }