1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import java.io.IOException;
21 import java.util.LinkedList;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.classification.InterfaceAudience;
26 import org.apache.hadoop.classification.InterfaceStability;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.Cell;
29 import org.apache.hadoop.hbase.DoNotRetryIOException;
30 import org.apache.hadoop.hbase.HBaseConfiguration;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.KeyValueUtil;
34 import org.apache.hadoop.hbase.NotServingRegionException;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.UnknownScannerException;
37 import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
38 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
39 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
40 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
41 import org.apache.hadoop.hbase.util.Bytes;
42
43
44
45
46
47
48 @InterfaceAudience.Public
49 @InterfaceStability.Stable
50 public class ClientScanner extends AbstractClientScanner {
51 private final Log LOG = LogFactory.getLog(this.getClass());
52 protected Scan scan;
53 protected boolean closed = false;
54
55
56 protected HRegionInfo currentRegion = null;
57 private ScannerCallable callable = null;
58 protected final LinkedList<Result> cache = new LinkedList<Result>();
59 protected final int caching;
60 protected long lastNext;
61
62 protected Result lastResult = null;
63 protected final long maxScannerResultSize;
64 private final HConnection connection;
65 private final TableName tableName;
66 private final int scannerTimeout;
67 protected boolean scanMetricsPublished = false;
68 protected RpcRetryingCaller<Result []> caller;
69
70
71
72
73
74
75
76
77
78
79
80 public ClientScanner(final Configuration conf, final Scan scan,
81 final TableName tableName) throws IOException {
82 this(conf, scan, tableName, HConnectionManager.getConnection(conf));
83 }
84
85
86
87
88 @Deprecated
89 public ClientScanner(final Configuration conf, final Scan scan,
90 final byte [] tableName) throws IOException {
91 this(conf, scan, TableName.valueOf(tableName));
92 }
93
94
95
96
97
98
99
100
101
102
103
104
105 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
106 HConnection connection) throws IOException {
107 this(conf, scan, tableName, connection, new RpcRetryingCallerFactory(conf));
108 }
109
110
111
112
113 @Deprecated
114 public ClientScanner(final Configuration conf, final Scan scan, final byte [] tableName,
115 HConnection connection) throws IOException {
116 this(conf, scan, TableName.valueOf(tableName), connection, new RpcRetryingCallerFactory(conf));
117 }
118
119
120
121
122
123
124
125
126
127
128 public ClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
129 HConnection connection, RpcRetryingCallerFactory rpcFactory) throws IOException {
130 if (LOG.isTraceEnabled()) {
131 LOG.trace("Scan table=" + tableName
132 + ", startRow=" + Bytes.toStringBinary(scan.getStartRow()));
133 }
134 this.scan = scan;
135 this.tableName = tableName;
136 this.lastNext = System.currentTimeMillis();
137 this.connection = connection;
138 if (scan.getMaxResultSize() > 0) {
139 this.maxScannerResultSize = scan.getMaxResultSize();
140 } else {
141 this.maxScannerResultSize = conf.getLong(
142 HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
143 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
144 }
145 this.scannerTimeout = HBaseConfiguration.getInt(conf,
146 HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
147 HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
148 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
149
150
151 initScanMetrics(scan);
152
153
154 if (this.scan.getCaching() > 0) {
155 this.caching = this.scan.getCaching();
156 } else {
157 this.caching = conf.getInt(
158 HConstants.HBASE_CLIENT_SCANNER_CACHING,
159 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
160 }
161
162 this.caller = rpcFactory.<Result[]> newCaller();
163
164 initializeScannerInConstruction();
165 }
166
167 protected void initializeScannerInConstruction() throws IOException{
168
169 nextScanner(this.caching, false);
170 }
171
172 protected HConnection getConnection() {
173 return this.connection;
174 }
175
176
177
178
179
180 @Deprecated
181 protected byte [] getTableName() {
182 return this.tableName.getName();
183 }
184
185 protected TableName getTable() {
186 return this.tableName;
187 }
188
189 protected Scan getScan() {
190 return scan;
191 }
192
193 protected long getTimestamp() {
194 return lastNext;
195 }
196
197
198 protected boolean checkScanStopRow(final byte [] endKey) {
199 if (this.scan.getStopRow().length > 0) {
200
201 byte [] stopRow = scan.getStopRow();
202 int cmp = Bytes.compareTo(stopRow, 0, stopRow.length,
203 endKey, 0, endKey.length);
204 if (cmp <= 0) {
205
206
207 return true;
208 }
209 }
210 return false;
211 }
212
213
214
215
216
217
218
219
220
221
222 private boolean nextScanner(int nbRows, final boolean done)
223 throws IOException {
224
225 if (this.callable != null) {
226 this.callable.setClose();
227 this.caller.callWithRetries(callable);
228 this.callable = null;
229 }
230
231
232 byte [] localStartKey;
233
234
235 if (this.currentRegion != null) {
236 byte [] endKey = this.currentRegion.getEndKey();
237 if (endKey == null ||
238 Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY) ||
239 checkScanStopRow(endKey) ||
240 done) {
241 close();
242 if (LOG.isTraceEnabled()) {
243 LOG.trace("Finished " + this.currentRegion);
244 }
245 return false;
246 }
247 localStartKey = endKey;
248 if (LOG.isTraceEnabled()) {
249 LOG.trace("Finished " + this.currentRegion);
250 }
251 } else {
252 localStartKey = this.scan.getStartRow();
253 }
254
255 if (LOG.isDebugEnabled() && this.currentRegion != null) {
256
257 LOG.debug("Advancing internal scanner to startKey at '" +
258 Bytes.toStringBinary(localStartKey) + "'");
259 }
260 try {
261 callable = getScannerCallable(localStartKey, nbRows);
262
263
264 this.caller.callWithRetries(callable);
265 this.currentRegion = callable.getHRegionInfo();
266 if (this.scanMetrics != null) {
267 this.scanMetrics.countOfRegions.incrementAndGet();
268 }
269 } catch (IOException e) {
270 close();
271 throw e;
272 }
273 return true;
274 }
275
276 @InterfaceAudience.Private
277 protected ScannerCallable getScannerCallable(byte [] localStartKey,
278 int nbRows) {
279 scan.setStartRow(localStartKey);
280 ScannerCallable s = new ScannerCallable(getConnection(),
281 getTable(), scan, this.scanMetrics);
282 s.setCaching(nbRows);
283 return s;
284 }
285
286
287
288
289
290
291
292
293
294
295
296
297 protected void writeScanMetrics() {
298 if (this.scanMetrics == null || scanMetricsPublished) {
299 return;
300 }
301 MapReduceProtos.ScanMetrics pScanMetrics = ProtobufUtil.toScanMetrics(scanMetrics);
302 scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, pScanMetrics.toByteArray());
303 scanMetricsPublished = true;
304 }
305
306 @Override
307 public Result next() throws IOException {
308
309 if (cache.size() == 0 && this.closed) {
310 return null;
311 }
312 if (cache.size() == 0) {
313 Result [] values = null;
314 long remainingResultSize = maxScannerResultSize;
315 int countdown = this.caching;
316
317
318 callable.setCaching(this.caching);
319
320
321 boolean skipFirst = false;
322 boolean retryAfterOutOfOrderException = true;
323 do {
324 try {
325 if (skipFirst) {
326
327
328 callable.setCaching(1);
329 values = this.caller.callWithRetries(callable);
330 callable.setCaching(this.caching);
331 skipFirst = false;
332 }
333
334
335
336 values = this.caller.callWithRetries(callable);
337 if (skipFirst && values != null && values.length == 1) {
338 skipFirst = false;
339 values = this.caller.callWithRetries(callable);
340 }
341 retryAfterOutOfOrderException = true;
342 } catch (DoNotRetryIOException e) {
343
344
345 if (e instanceof UnknownScannerException) {
346 long timeout = lastNext + scannerTimeout;
347
348
349
350 if (timeout < System.currentTimeMillis()) {
351 long elapsed = System.currentTimeMillis() - lastNext;
352 ScannerTimeoutException ex = new ScannerTimeoutException(
353 elapsed + "ms passed since the last invocation, " +
354 "timeout is currently set to " + scannerTimeout);
355 ex.initCause(e);
356 throw ex;
357 }
358 } else {
359
360
361 Throwable cause = e.getCause();
362 if ((cause != null && cause instanceof NotServingRegionException) ||
363 (cause != null && cause instanceof RegionServerStoppedException) ||
364 e instanceof OutOfOrderScannerNextException) {
365
366
367
368 } else {
369 throw e;
370 }
371 }
372
373 if (this.lastResult != null) {
374 this.scan.setStartRow(this.lastResult.getRow());
375
376
377 skipFirst = true;
378 }
379 if (e instanceof OutOfOrderScannerNextException) {
380 if (retryAfterOutOfOrderException) {
381 retryAfterOutOfOrderException = false;
382 } else {
383
384 throw new DoNotRetryIOException("Failed after retry of " +
385 "OutOfOrderScannerNextException: was there a rpc timeout?", e);
386 }
387 }
388
389 this.currentRegion = null;
390
391
392 callable = null;
393
394 continue;
395 }
396 long currentTime = System.currentTimeMillis();
397 if (this.scanMetrics != null ) {
398 this.scanMetrics.sumOfMillisSecBetweenNexts.addAndGet(currentTime-lastNext);
399 }
400 lastNext = currentTime;
401 if (values != null && values.length > 0) {
402 for (Result rs : values) {
403 cache.add(rs);
404 for (Cell kv : rs.rawCells()) {
405
406 remainingResultSize -= KeyValueUtil.ensureKeyValue(kv).heapSize();
407 }
408 countdown--;
409 this.lastResult = rs;
410 }
411 }
412
413 } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
414 }
415
416 if (cache.size() > 0) {
417 return cache.poll();
418 }
419
420
421 writeScanMetrics();
422 return null;
423 }
424
425 @Override
426 public void close() {
427 if (!scanMetricsPublished) writeScanMetrics();
428 if (callable != null) {
429 callable.setClose();
430 try {
431 this.caller.callWithRetries(callable);
432 } catch (IOException e) {
433
434
435
436
437 }
438 callable = null;
439 }
440 closed = true;
441 }
442 }