1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.regionserver;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.NavigableSet;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.DoNotRetryIOException;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.client.Scan;
34 import org.apache.hadoop.hbase.filter.Filter;
35 import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
36 import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage;
37 import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
40
41
42
43
44
45 public class StoreScanner extends NonLazyKeyValueScanner
46 implements KeyValueScanner, InternalScanner, ChangedReadersObserver {
47 static final Log LOG = LogFactory.getLog(StoreScanner.class);
48 private Store store;
49 private ScanQueryMatcher matcher;
50 private KeyValueHeap heap;
51 private boolean cacheBlocks;
52
53
54 private String metricNamePrefix;
55
56
57 private boolean closing = false;
58 private final boolean isGet;
59 private final boolean explicitColumnQuery;
60 private final boolean useRowColBloom;
61 private final Scan scan;
62 private final NavigableSet<byte[]> columns;
63 private final long oldestUnexpiredTS;
64 private final int minVersions;
65
66
67 static final boolean LAZY_SEEK_ENABLED_BY_DEFAULT = true;
68
69
70 private static boolean lazySeekEnabledGlobally =
71 LAZY_SEEK_ENABLED_BY_DEFAULT;
72
73
74 private KeyValue lastTop = null;
75
76
77 private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
78 final NavigableSet<byte[]> columns, long ttl, int minVersions) {
79 this.store = store;
80 this.cacheBlocks = cacheBlocks;
81 isGet = scan.isGetScan();
82 int numCol = columns == null ? 0 : columns.size();
83 explicitColumnQuery = numCol > 0;
84 this.scan = scan;
85 this.columns = columns;
86 oldestUnexpiredTS = EnvironmentEdgeManager.currentTimeMillis() - ttl;
87 this.minVersions = minVersions;
88
89
90
91
92
93 useRowColBloom = numCol > 1 || (!isGet && numCol == 1);
94 }
95
96
97
98
99
100
101
102
103
104
105 public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, final NavigableSet<byte[]> columns)
106 throws IOException {
107 this(store, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(),
108 scanInfo.getMinVersions());
109 initializeMetricNames();
110 if (columns != null && scan.isRaw()) {
111 throw new DoNotRetryIOException(
112 "Cannot specify any column for a raw scan");
113 }
114 matcher = new ScanQueryMatcher(scan, scanInfo, columns,
115 ScanType.USER_SCAN, Long.MAX_VALUE, HConstants.LATEST_TIMESTAMP,
116 oldestUnexpiredTS);
117
118
119 List<KeyValueScanner> scanners = getScannersNoCompaction();
120
121
122
123
124
125 if (explicitColumnQuery && lazySeekEnabledGlobally) {
126 for (KeyValueScanner scanner : scanners) {
127 scanner.requestSeek(matcher.getStartKey(), false, true);
128 }
129 } else {
130 for (KeyValueScanner scanner : scanners) {
131 scanner.seek(matcher.getStartKey());
132 }
133 }
134
135
136 heap = new KeyValueHeap(scanners, store.comparator);
137
138 this.store.addChangedReaderObserver(this);
139 }
140
141
142
143
144
145
146
147
148
149
150
151 public StoreScanner(Store store, ScanInfo scanInfo, Scan scan,
152 List<? extends KeyValueScanner> scanners, ScanType scanType,
153 long smallestReadPoint, long earliestPutTs) throws IOException {
154 this(store, false, scan, null, scanInfo.getTtl(),
155 scanInfo.getMinVersions());
156 initializeMetricNames();
157 matcher = new ScanQueryMatcher(scan, scanInfo, null, scanType,
158 smallestReadPoint, earliestPutTs, oldestUnexpiredTS);
159
160
161 scanners = selectScannersFrom(scanners);
162
163
164 for(KeyValueScanner scanner : scanners) {
165 scanner.seek(matcher.getStartKey());
166 }
167
168
169 heap = new KeyValueHeap(scanners, store.comparator);
170 }
171
172
173 StoreScanner(final Scan scan, Store.ScanInfo scanInfo,
174 ScanType scanType, final NavigableSet<byte[]> columns,
175 final List<KeyValueScanner> scanners) throws IOException {
176 this(scan, scanInfo, scanType, columns, scanners,
177 HConstants.LATEST_TIMESTAMP);
178 }
179
180
181 StoreScanner(final Scan scan, Store.ScanInfo scanInfo,
182 ScanType scanType, final NavigableSet<byte[]> columns,
183 final List<KeyValueScanner> scanners, long earliestPutTs)
184 throws IOException {
185 this(null, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(),
186 scanInfo.getMinVersions());
187 this.initializeMetricNames();
188 this.matcher = new ScanQueryMatcher(scan, scanInfo, columns, scanType,
189 Long.MAX_VALUE, earliestPutTs, oldestUnexpiredTS);
190
191
192 for (KeyValueScanner scanner : scanners) {
193 scanner.seek(matcher.getStartKey());
194 }
195 heap = new KeyValueHeap(scanners, scanInfo.getComparator());
196 }
197
198
199
200
201
202
203
204 private void initializeMetricNames() {
205 String tableName = SchemaMetrics.UNKNOWN;
206 String family = SchemaMetrics.UNKNOWN;
207 if (store != null) {
208 tableName = store.getTableName();
209 family = Bytes.toString(store.getFamily().getName());
210 }
211 this.metricNamePrefix =
212 SchemaMetrics.generateSchemaMetricsPrefix(tableName, family);
213 }
214
215
216
217
218
219 private List<KeyValueScanner> getScannersNoCompaction() throws IOException {
220 final boolean isCompaction = false;
221 return selectScannersFrom(store.getScanners(cacheBlocks, isGet,
222 isCompaction, matcher));
223 }
224
225
226
227
228
229 private List<KeyValueScanner> selectScannersFrom(
230 final List<? extends KeyValueScanner> allScanners) {
231 boolean memOnly;
232 boolean filesOnly;
233 if (scan instanceof InternalScan) {
234 InternalScan iscan = (InternalScan)scan;
235 memOnly = iscan.isCheckOnlyMemStore();
236 filesOnly = iscan.isCheckOnlyStoreFiles();
237 } else {
238 memOnly = false;
239 filesOnly = false;
240 }
241
242 List<KeyValueScanner> scanners =
243 new ArrayList<KeyValueScanner>(allScanners.size());
244
245
246
247 long expiredTimestampCutoff = minVersions == 0 ? oldestUnexpiredTS :
248 Long.MIN_VALUE;
249
250
251 for (KeyValueScanner kvs : allScanners) {
252 boolean isFile = kvs.isFileScanner();
253 if ((!isFile && filesOnly) || (isFile && memOnly)) {
254 continue;
255 }
256
257 if (kvs.shouldUseScanner(scan, columns, expiredTimestampCutoff)) {
258 scanners.add(kvs);
259 }
260 }
261 return scanners;
262 }
263
264 @Override
265 public synchronized KeyValue peek() {
266 if (this.heap == null) {
267 return this.lastTop;
268 }
269 return this.heap.peek();
270 }
271
272 @Override
273 public KeyValue next() {
274
275 throw new RuntimeException("Never call StoreScanner.next()");
276 }
277
278 @Override
279 public synchronized void close() {
280 if (this.closing) return;
281 this.closing = true;
282
283 if (this.store != null)
284 this.store.deleteChangedReaderObserver(this);
285 if (this.heap != null)
286 this.heap.close();
287 this.heap = null;
288 this.lastTop = null;
289 }
290
291 @Override
292 public synchronized boolean seek(KeyValue key) throws IOException {
293 if (this.heap == null) {
294
295 List<KeyValueScanner> scanners = getScannersNoCompaction();
296
297 heap = new KeyValueHeap(scanners, store.comparator);
298 }
299
300 return this.heap.seek(key);
301 }
302
303
304
305
306
307
308
309 @Override
310 public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOException {
311 return next(outResult, limit, null);
312 }
313
314
315
316
317
318
319
320 @Override
321 public synchronized boolean next(List<KeyValue> outResult, int limit,
322 String metric) throws IOException {
323
324 if (checkReseek()) {
325 return true;
326 }
327
328
329
330 if (this.heap == null) {
331 close();
332 return false;
333 }
334
335 KeyValue peeked = this.heap.peek();
336 if (peeked == null) {
337 close();
338 return false;
339 }
340
341
342
343 byte[] row = peeked.getBuffer();
344 int offset = peeked.getRowOffset();
345 short length = peeked.getRowLength();
346 if ((matcher.row == null) || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) {
347 matcher.setRow(row, offset, length);
348 }
349
350 KeyValue kv;
351 KeyValue prevKV = null;
352
353
354 KeyValue.KVComparator comparator =
355 store != null ? store.getComparator() : null;
356
357 long cumulativeMetric = 0;
358 int count = 0;
359 try {
360 LOOP: while((kv = this.heap.peek()) != null) {
361
362 assert prevKV == null || comparator == null || comparator.compare(prevKV, kv) <= 0 :
363 "Key " + prevKV + " followed by a " + "smaller key " + kv + " in cf " + store;
364 prevKV = kv;
365 ScanQueryMatcher.MatchCode qcode = matcher.match(kv);
366 switch(qcode) {
367 case INCLUDE:
368 case INCLUDE_AND_SEEK_NEXT_ROW:
369 case INCLUDE_AND_SEEK_NEXT_COL:
370
371 Filter f = matcher.getFilter();
372 outResult.add(f == null ? kv : f.transform(kv));
373 count++;
374
375 if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW) {
376 if (!matcher.moreRowsMayExistAfter(kv)) {
377 return false;
378 }
379 reseek(matcher.getKeyForNextRow(kv));
380 } else if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL) {
381 reseek(matcher.getKeyForNextColumn(kv));
382 } else {
383 this.heap.next();
384 }
385
386 cumulativeMetric += kv.getLength();
387 if (limit > 0 && (count == limit)) {
388 break LOOP;
389 }
390 continue;
391
392 case DONE:
393 return true;
394
395 case DONE_SCAN:
396 close();
397
398 return false;
399
400 case SEEK_NEXT_ROW:
401
402
403 if (!matcher.moreRowsMayExistAfter(kv)) {
404 return false;
405 }
406
407 reseek(matcher.getKeyForNextRow(kv));
408 break;
409
410 case SEEK_NEXT_COL:
411 reseek(matcher.getKeyForNextColumn(kv));
412 break;
413
414 case SKIP:
415 this.heap.next();
416 break;
417
418 case SEEK_NEXT_USING_HINT:
419 KeyValue nextKV = matcher.getNextKeyHint(kv);
420 if (nextKV != null) {
421 reseek(nextKV);
422 } else {
423 heap.next();
424 }
425 break;
426
427 default:
428 throw new RuntimeException("UNEXPECTED");
429 }
430 }
431 } finally {
432 if (cumulativeMetric > 0 && metric != null) {
433 RegionMetricsStorage.incrNumericMetric(this.metricNamePrefix + metric,
434 cumulativeMetric);
435 }
436 }
437
438 if (count > 0) {
439 return true;
440 }
441
442
443 close();
444 return false;
445 }
446
447 @Override
448 public synchronized boolean next(List<KeyValue> outResult) throws IOException {
449 return next(outResult, -1, null);
450 }
451
452 @Override
453 public synchronized boolean next(List<KeyValue> outResult, String metric)
454 throws IOException {
455 return next(outResult, -1, metric);
456 }
457
458
459 @Override
460 public synchronized void updateReaders() throws IOException {
461 if (this.closing) return;
462
463
464
465
466
467
468 if (this.heap == null) return;
469
470
471 this.lastTop = this.peek();
472
473
474
475
476 this.heap.close();
477 this.heap = null;
478
479
480 }
481
482
483
484
485
486
487 private boolean checkReseek() throws IOException {
488 if (this.heap == null && this.lastTop != null) {
489 resetScannerStack(this.lastTop);
490 if (this.heap.peek() == null
491 || store.comparator.compareRows(this.lastTop, this.heap.peek()) != 0) {
492 LOG.debug("Storescanner.peek() is changed where before = "
493 + this.lastTop.toString() + ",and after = " + this.heap.peek());
494 this.lastTop = null;
495 return true;
496 }
497 this.lastTop = null;
498 }
499
500 return false;
501 }
502
503 private void resetScannerStack(KeyValue lastTopKey) throws IOException {
504 if (heap != null) {
505 throw new RuntimeException("StoreScanner.reseek run on an existing heap!");
506 }
507
508
509
510
511 List<KeyValueScanner> scanners = getScannersNoCompaction();
512
513 for(KeyValueScanner scanner : scanners) {
514 scanner.seek(lastTopKey);
515 }
516
517
518 heap = new KeyValueHeap(scanners, store.comparator);
519
520
521
522
523 KeyValue kv = heap.peek();
524 if (kv == null) {
525 kv = lastTopKey;
526 }
527 byte[] row = kv.getBuffer();
528 int offset = kv.getRowOffset();
529 short length = kv.getRowLength();
530 if ((matcher.row == null) || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) {
531 matcher.reset();
532 matcher.setRow(row, offset, length);
533 }
534 }
535
536 @Override
537 public synchronized boolean reseek(KeyValue kv) throws IOException {
538
539
540
541 checkReseek();
542 if (explicitColumnQuery && lazySeekEnabledGlobally) {
543 return heap.requestSeek(kv, true, useRowColBloom);
544 } else {
545 return heap.reseek(kv);
546 }
547 }
548
549 @Override
550 public long getSequenceID() {
551 return 0;
552 }
553
554
555
556
557
558 List<KeyValueScanner> getAllScannersForTesting() {
559 List<KeyValueScanner> allScanners = new ArrayList<KeyValueScanner>();
560 KeyValueScanner current = heap.getCurrentForTesting();
561 if (current != null)
562 allScanners.add(current);
563 for (KeyValueScanner scanner : heap.getHeap())
564 allScanners.add(scanner);
565 return allScanners;
566 }
567
568 static void enableLazySeekGlobally(boolean enable) {
569 lazySeekEnabledGlobally = enable;
570 }
571 }
572