1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import java.io.IOException;
21 import java.util.LinkedList;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.hbase.classification.InterfaceStability;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.Cell;
29 import org.apache.hadoop.hbase.DoNotRetryIOException;
30 import org.apache.hadoop.hbase.HBaseConfiguration;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.KeyValueUtil;
34 import org.apache.hadoop.hbase.NotServingRegionException;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.UnknownScannerException;
37 import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
38 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
41 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
42 import org.apache.hadoop.hbase.util.Bytes;
43
44
45
46
47
48
49 @InterfaceAudience.Public
50 @InterfaceStability.Stable
51 public class ClientScanner extends AbstractClientScanner {
52 private final Log LOG = LogFactory.getLog(this.getClass());
53 protected Scan scan;
54 protected boolean closed = false;
55
56
57 protected HRegionInfo currentRegion = null;
58 protected ScannerCallable callable = null;
59 protected final LinkedList<Result> cache = new LinkedList<Result>();
60 protected final int caching;
61 protected long lastNext;
62
63 protected Result lastResult = null;
64 protected final long maxScannerResultSize;
65 private final HConnection connection;
66 private final TableName tableName;
67 protected final int scannerTimeout;
68 protected boolean scanMetricsPublished = false;
69 protected RpcRetryingCaller<Result []> caller;
70 protected RpcControllerFactory rpcControllerFactory;
71
72
73
74
75
76
77
78
79
80
81
82 @Deprecated
83 public ClientScanner(final Configuration conf, final Scan scan,
84 final TableName tableName) throws IOException {
85 this(conf, scan, tableName, HConnectionManager.getConnection(conf));
86 }
87
88
89
90
91 @Deprecated
92 public ClientScanner(final Configuration conf, final Scan scan,
93 final byte [] tableName) throws IOException {
94 this(conf, scan, TableName.valueOf(tableName));
95 }
96
97
98
99
100
101
102
103
104
105
106
107
108 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
109 HConnection connection) throws IOException {
110 this(conf, scan, tableName, connection, RpcRetryingCallerFactory.instantiate(conf),
111 RpcControllerFactory.instantiate(conf));
112 }
113
114
115
116
117 @Deprecated
118 public ClientScanner(final Configuration conf, final Scan scan, final byte [] tableName,
119 HConnection connection) throws IOException {
120 this(conf, scan, TableName.valueOf(tableName), connection, new RpcRetryingCallerFactory(conf),
121 RpcControllerFactory.instantiate(conf));
122 }
123
124
125
126
127
128
129
130 @Deprecated
131 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
132 HConnection connection, RpcRetryingCallerFactory rpcFactory) throws IOException {
133 this(conf, scan, tableName, connection, rpcFactory, RpcControllerFactory.instantiate(conf));
134 }
135
136
137
138
139
140
141
142
143
144
145 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
146 HConnection connection, RpcRetryingCallerFactory rpcFactory,
147 RpcControllerFactory controllerFactory) throws IOException {
148 if (LOG.isTraceEnabled()) {
149 LOG.trace("Scan table=" + tableName
150 + ", startRow=" + Bytes.toStringBinary(scan.getStartRow()));
151 }
152 this.scan = scan;
153 this.tableName = tableName;
154 this.lastNext = System.currentTimeMillis();
155 this.connection = connection;
156 if (scan.getMaxResultSize() > 0) {
157 this.maxScannerResultSize = scan.getMaxResultSize();
158 } else {
159 this.maxScannerResultSize = conf.getLong(
160 HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
161 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
162 }
163 this.scannerTimeout = HBaseConfiguration.getInt(conf,
164 HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
165 HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
166 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
167
168
169 initScanMetrics(scan);
170
171
172 if (this.scan.getCaching() > 0) {
173 this.caching = this.scan.getCaching();
174 } else {
175 this.caching = conf.getInt(
176 HConstants.HBASE_CLIENT_SCANNER_CACHING,
177 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
178 }
179
180 this.caller = rpcFactory.<Result[]> newCaller();
181 this.rpcControllerFactory = controllerFactory;
182
183 initializeScannerInConstruction();
184 }
185
186 protected void initializeScannerInConstruction() throws IOException{
187
188 nextScanner(this.caching, false);
189 }
190
191 protected HConnection getConnection() {
192 return this.connection;
193 }
194
195
196
197
198
199 @Deprecated
200 protected byte [] getTableName() {
201 return this.tableName.getName();
202 }
203
204 protected TableName getTable() {
205 return this.tableName;
206 }
207
208 protected Scan getScan() {
209 return scan;
210 }
211
212 protected long getTimestamp() {
213 return lastNext;
214 }
215
216
217 protected boolean checkScanStopRow(final byte [] endKey) {
218 if (this.scan.getStopRow().length > 0) {
219
220 byte [] stopRow = scan.getStopRow();
221 int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
222 endKey, 0, endKey.length);
223 if (cmp <= 0) {
224
225
226 return true;
227 }
228 }
229 return false;
230 }
231
232
233
234
235
236
237
238
239
240
241 protected boolean nextScanner(int nbRows, final boolean done)
242 throws IOException {
243
244 if (this.callable != null) {
245 this.callable.setClose();
246 this.caller.callWithRetries(callable);
247 this.callable = null;
248 }
249
250
251 byte [] localStartKey;
252
253
254 if (this.currentRegion != null) {
255 byte [] endKey = this.currentRegion.getEndKey();
256 if (endKey == null ||
257 Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
258 checkScanStopRow(endKey) ||
259 done) {
260 close();
261 if (LOG.isTraceEnabled()) {
262 LOG.trace("Finished " + this.currentRegion);
263 }
264 return false;
265 }
266 localStartKey = endKey;
267 if (LOG.isTraceEnabled()) {
268 LOG.trace("Finished " + this.currentRegion);
269 }
270 } else {
271 localStartKey = this.scan.getStartRow();
272 }
273
274 if (LOG.isDebugEnabled() && this.currentRegion != null) {
275
276 LOG.debug("Advancing internal scanner to startKey at '" +
277 Bytes.toStringBinary(localStartKey) + "'");
278 }
279 try {
280 callable = getScannerCallable(localStartKey, nbRows);
281
282
283 this.caller.callWithRetries(callable);
284 this.currentRegion = callable.getHRegionInfo();
285 if (this.scanMetrics != null) {
286 this.scanMetrics.countOfRegions.incrementAndGet();
287 }
288 } catch (IOException e) {
289 close();
290 throw e;
291 }
292 return true;
293 }
294
295 @InterfaceAudience.Private
296 protected ScannerCallable getScannerCallable(byte [] localStartKey,
297 int nbRows) {
298 scan.setStartRow(localStartKey);
299 ScannerCallable s = new ScannerCallable(getConnection(),
300 getTable(), scan, this.scanMetrics, rpcControllerFactory.newController());
301 s.setCaching(nbRows);
302 return s;
303 }
304
305
306
307
308
309
310
311
312
313
314
315
316 protected void writeScanMetrics() {
317 if (this.scanMetrics == null || scanMetricsPublished) {
318 return;
319 }
320 MapReduceProtos.ScanMetrics pScanMetrics = ProtobufUtil.toScanMetrics(scanMetrics);
321 scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, pScanMetrics.toByteArray());
322 scanMetricsPublished = true;
323 }
324
325 @Override
326 public Result next() throws IOException {
327
328 if (cache.size() == 0 && this.closed) {
329 return null;
330 }
331 if (cache.size() == 0) {
332 Result [] values = null;
333 long remainingResultSize = maxScannerResultSize;
334 int countdown = this.caching;
335
336
337 callable.setCaching(this.caching);
338
339
340 boolean skipFirst = false;
341 boolean retryAfterOutOfOrderException = true;
342 do {
343 try {
344 if (skipFirst) {
345
346
347 callable.setCaching(1);
348 values = this.caller.callWithRetries(callable);
349 callable.setCaching(this.caching);
350 skipFirst = false;
351 }
352
353
354
355 values = this.caller.callWithRetries(callable);
356 if (skipFirst && values != null && values.length == 1) {
357 skipFirst = false;
358 values = this.caller.callWithRetries(callable);
359 }
360 retryAfterOutOfOrderException = true;
361 } catch (DoNotRetryIOException e) {
362
363
364 if (e instanceof UnknownScannerException) {
365 long timeout = lastNext + scannerTimeout;
366
367
368
369 if (timeout < System.currentTimeMillis()) {
370 long elapsed = System.currentTimeMillis() - lastNext;
371 ScannerTimeoutException ex = new ScannerTimeoutException(
372 elapsed + "ms passed since the last invocation, " +
373 "timeout is currently set to " + scannerTimeout);
374 ex.initCause(e);
375 throw ex;
376 }
377 } else {
378
379
380 Throwable cause = e.getCause();
381 if ((cause != null && cause instanceof NotServingRegionException) ||
382 (cause != null && cause instanceof RegionServerStoppedException) ||
383 e instanceof OutOfOrderScannerNextException) {
384
385
386
387 } else {
388 throw e;
389 }
390 }
391
392 if (this.lastResult != null) {
393
394
395
396
397
398
399 this.scan.setStartRow(this.lastResult.getRow());
400
401
402
403 skipFirst = true;
404 }
405 if (e instanceof OutOfOrderScannerNextException) {
406 if (retryAfterOutOfOrderException) {
407 retryAfterOutOfOrderException = false;
408 } else {
409
410 throw new DoNotRetryIOException("Failed after retry of " +
411 "OutOfOrderScannerNextException: was there a rpc timeout?", e);
412 }
413 }
414
415 this.currentRegion = null;
416
417
418 callable = null;
419
420 continue;
421 }
422 long currentTime = System.currentTimeMillis();
423 if (this.scanMetrics != null ) {
424 this.scanMetrics.sumOfMillisSecBetweenNexts.addAndGet(currentTime-lastNext);
425 }
426 lastNext = currentTime;
427 if (values != null && values.length > 0) {
428 for (Result rs : values) {
429 cache.add(rs);
430 for (Cell kv : rs.rawCells()) {
431
432 remainingResultSize -= KeyValueUtil.ensureKeyValue(kv).heapSize();
433 }
434 countdown--;
435 this.lastResult = rs;
436 }
437 }
438
439 } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
440 }
441
442 if (cache.size() > 0) {
443 return cache.poll();
444 }
445
446
447 writeScanMetrics();
448 return null;
449 }
450
451 @Override
452 public void close() {
453 if (!scanMetricsPublished) writeScanMetrics();
454 if (callable != null) {
455 callable.setClose();
456 try {
457 this.caller.callWithRetries(callable);
458 } catch (UnknownScannerException e) {
459
460
461
462 } catch (IOException e) {
463
464 LOG.warn("scanner failed to close. Exception follows: " + e);
465 }
466 callable = null;
467 }
468 closed = true;
469 }
470 }