1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.NavigableSet;
24
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.hbase.Cell;
27 import org.apache.hadoop.hbase.HConstants;
28 import org.apache.hadoop.hbase.KeepDeletedCells;
29 import org.apache.hadoop.hbase.KeyValue;
30 import org.apache.hadoop.hbase.client.Scan;
31 import org.apache.hadoop.hbase.filter.Filter;
32 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
33 import org.apache.hadoop.hbase.io.TimeRange;
34 import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
37
38 import com.google.common.base.Preconditions;
39
40
41
42
43 @InterfaceAudience.Private
44 public class ScanQueryMatcher {
45
46
47 private boolean stickyNextRow;
48 private final byte[] stopRow;
49
50 private final TimeRange tr;
51
52 private final Filter filter;
53
54
55 private final DeleteTracker deletes;
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71 private boolean retainDeletesInOutput;
72
73
74 private final KeepDeletedCells keepDeletedCells;
75
76 private final boolean seePastDeleteMarkers;
77
78
79
80 private final ColumnTracker columns;
81
82
83 private final KeyValue startKey;
84
85
86 private final KeyValue.KVComparator rowComparator;
87
88
89
90 byte [] row;
91 int rowOffset;
92 short rowLength;
93
94
95
96
97
98
99
100 private final long earliestPutTs;
101 private final long ttl;
102
103
104 protected long maxReadPointToTrackVersions;
105
106 private byte[] dropDeletesFromRow = null, dropDeletesToRow = null;
107
108
109
110
111
112
113
114 private boolean hasNullColumn = true;
115
116 private RegionCoprocessorHost regionCoprocessorHost= null;
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135 private final long timeToPurgeDeletes;
136
137 private final boolean isUserScan;
138
139 private final boolean isReversed;
140
141
142
143
144
145
146
147
148
149
150
151
152
153 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
154 ScanType scanType, long readPointToUse, long earliestPutTs, long oldestUnexpiredTS,
155 RegionCoprocessorHost regionCoprocessorHost) throws IOException {
156 this.tr = scan.getTimeRange();
157 this.rowComparator = scanInfo.getComparator();
158 this.regionCoprocessorHost = regionCoprocessorHost;
159 this.deletes = instantiateDeleteTracker();
160 this.stopRow = scan.getStopRow();
161 this.startKey = KeyValue.createFirstDeleteFamilyOnRow(scan.getStartRow(),
162 scanInfo.getFamily());
163 this.filter = scan.getFilter();
164 this.earliestPutTs = earliestPutTs;
165 this.maxReadPointToTrackVersions = readPointToUse;
166 this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes();
167 this.ttl = oldestUnexpiredTS;
168
169
170 this.isUserScan = scanType == ScanType.USER_SCAN;
171
172 this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE :
173 isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells();
174
175 this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw();
176
177 this.seePastDeleteMarkers =
178 scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan;
179
180 int maxVersions =
181 scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(),
182 scanInfo.getMaxVersions());
183
184
185 if (columns == null || columns.size() == 0) {
186
187 hasNullColumn = true;
188
189
190 this.columns = new ScanWildcardColumnTracker(
191 scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS);
192 } else {
193
194 hasNullColumn = (columns.first().length == 0);
195
196
197
198 byte[] attr = scan.getAttribute(Scan.HINT_LOOKAHEAD);
199 this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions,
200 oldestUnexpiredTS, attr == null ? 0 : Bytes.toInt(attr));
201 }
202 this.isReversed = scan.isReversed();
203 }
204
205 private DeleteTracker instantiateDeleteTracker() throws IOException {
206 DeleteTracker tracker = new ScanDeleteTracker();
207 if (regionCoprocessorHost != null) {
208 tracker = regionCoprocessorHost.postInstantiateDeleteTracker(tracker);
209 }
210 return tracker;
211 }
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
227 long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, byte[] dropDeletesFromRow,
228 byte[] dropDeletesToRow, RegionCoprocessorHost regionCoprocessorHost) throws IOException {
229 this(scan, scanInfo, columns, ScanType.COMPACT_RETAIN_DELETES, readPointToUse, earliestPutTs,
230 oldestUnexpiredTS, regionCoprocessorHost);
231 Preconditions.checkArgument((dropDeletesFromRow != null) && (dropDeletesToRow != null));
232 this.dropDeletesFromRow = dropDeletesFromRow;
233 this.dropDeletesToRow = dropDeletesToRow;
234 }
235
236
237
238
239 ScanQueryMatcher(Scan scan, ScanInfo scanInfo,
240 NavigableSet<byte[]> columns, long oldestUnexpiredTS) throws IOException {
241 this(scan, scanInfo, columns, ScanType.USER_SCAN,
242 Long.MAX_VALUE,
243 HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS, null);
244 }
245
246
247
248
249
250 public boolean hasNullColumnInQuery() {
251 return hasNullColumn;
252 }
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267 public MatchCode match(KeyValue kv) throws IOException {
268 if (filter != null && filter.filterAllRemaining()) {
269 return MatchCode.DONE_SCAN;
270 }
271
272 byte [] bytes = kv.getBuffer();
273 int offset = kv.getOffset();
274
275 int keyLength = Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
276 offset += KeyValue.ROW_OFFSET;
277
278 int initialOffset = offset;
279
280 short rowLength = Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT);
281 offset += Bytes.SIZEOF_SHORT;
282
283 int ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength,
284 bytes, offset, rowLength);
285 if (!this.isReversed) {
286 if (ret <= -1) {
287 return MatchCode.DONE;
288 } else if (ret >= 1) {
289
290
291
292 return MatchCode.SEEK_NEXT_ROW;
293 }
294 } else {
295 if (ret <= -1) {
296 return MatchCode.SEEK_NEXT_ROW;
297 } else if (ret >= 1) {
298 return MatchCode.DONE;
299 }
300 }
301
302
303
304 if (this.stickyNextRow)
305 return MatchCode.SEEK_NEXT_ROW;
306
307 if (this.columns.done()) {
308 stickyNextRow = true;
309 return MatchCode.SEEK_NEXT_ROW;
310 }
311
312
313 offset += rowLength;
314
315
316 byte familyLength = bytes [offset];
317 offset += familyLength + 1;
318
319 int qualLength = keyLength -
320 (offset - initialOffset) - KeyValue.TIMESTAMP_TYPE_SIZE;
321
322 long timestamp = Bytes.toLong(bytes, initialOffset + keyLength - KeyValue.TIMESTAMP_TYPE_SIZE);
323
324 if (columns.isDone(timestamp)) {
325 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
326 }
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341 byte type = bytes[initialOffset + keyLength - 1];
342 if (kv.isDelete()) {
343 if (keepDeletedCells == KeepDeletedCells.FALSE
344 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) {
345
346
347
348
349
350
351 boolean includeDeleteMarker = seePastDeleteMarkers ?
352 tr.withinTimeRange(timestamp) :
353 tr.withinOrAfterTimeRange(timestamp);
354 if (includeDeleteMarker
355 && kv.getMvccVersion() <= maxReadPointToTrackVersions) {
356 this.deletes.add(kv);
357 }
358
359 }
360
361 if ((!isUserScan)
362 && timeToPurgeDeletes > 0
363 && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes) {
364 return MatchCode.INCLUDE;
365 } else if (retainDeletesInOutput || kv.getMvccVersion() > maxReadPointToTrackVersions) {
366
367
368 if (!isUserScan) {
369
370
371 return MatchCode.INCLUDE;
372 }
373 } else if (keepDeletedCells == KeepDeletedCells.TRUE
374 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) {
375 if (timestamp < earliestPutTs) {
376
377
378 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
379 }
380
381
382 } else {
383 return MatchCode.SKIP;
384 }
385
386
387 } else if (!this.deletes.isEmpty()) {
388 DeleteResult deleteResult = deletes.isDeleted(kv);
389 switch (deleteResult) {
390 case FAMILY_DELETED:
391 case COLUMN_DELETED:
392 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
393 case VERSION_DELETED:
394 case FAMILY_VERSION_DELETED:
395 return MatchCode.SKIP;
396 case NOT_DELETED:
397 break;
398 default:
399 throw new RuntimeException("UNEXPECTED");
400 }
401 }
402
403 int timestampComparison = tr.compare(timestamp);
404 if (timestampComparison >= 1) {
405 return MatchCode.SKIP;
406 } else if (timestampComparison <= -1) {
407 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
408 }
409
410
411 MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength, type);
412 if (colChecker == MatchCode.INCLUDE) {
413 ReturnCode filterResponse = ReturnCode.SKIP;
414
415 if (filter != null) {
416
417 filterResponse = filter.filterKeyValue(kv);
418 switch (filterResponse) {
419 case SKIP:
420 return MatchCode.SKIP;
421 case NEXT_COL:
422 return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
423 case NEXT_ROW:
424 stickyNextRow = true;
425 return MatchCode.SEEK_NEXT_ROW;
426 case SEEK_NEXT_USING_HINT:
427 return MatchCode.SEEK_NEXT_USING_HINT;
428 default:
429
430 break;
431 }
432 }
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452 colChecker =
453 columns.checkVersions(bytes, offset, qualLength, timestamp, type,
454 kv.getMvccVersion() > maxReadPointToTrackVersions);
455
456 stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow;
457 return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL &&
458 colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL
459 : colChecker;
460 }
461 stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true
462 : stickyNextRow;
463 return colChecker;
464 }
465
466
467
468
469 private void checkPartialDropDeleteRange(byte [] row, int offset, short length) {
470
471
472
473
474 if ((dropDeletesFromRow != null)
475 && ((dropDeletesFromRow == HConstants.EMPTY_START_ROW)
476 || (Bytes.compareTo(row, offset, length,
477 dropDeletesFromRow, 0, dropDeletesFromRow.length) >= 0))) {
478 retainDeletesInOutput = false;
479 dropDeletesFromRow = null;
480 }
481
482
483
484 if ((dropDeletesFromRow == null)
485 && (dropDeletesToRow != null) && (dropDeletesToRow != HConstants.EMPTY_END_ROW)
486 && (Bytes.compareTo(row, offset, length,
487 dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) {
488 retainDeletesInOutput = true;
489 dropDeletesToRow = null;
490 }
491 }
492
493 public boolean moreRowsMayExistAfter(KeyValue kv) {
494 if (this.isReversed) {
495 if (rowComparator.compareRows(kv.getBuffer(), kv.getRowOffset(),
496 kv.getRowLength(), stopRow, 0, stopRow.length) <= 0) {
497 return false;
498 } else {
499 return true;
500 }
501 }
502 if (!Bytes.equals(stopRow , HConstants.EMPTY_END_ROW) &&
503 rowComparator.compareRows(kv.getBuffer(),kv.getRowOffset(),
504 kv.getRowLength(), stopRow, 0, stopRow.length) >= 0) {
505
506
507 return false;
508 } else {
509 return true;
510 }
511 }
512
513
514
515
516
517 public void setRow(byte [] row, int offset, short length) {
518 checkPartialDropDeleteRange(row, offset, length);
519 this.row = row;
520 this.rowOffset = offset;
521 this.rowLength = length;
522 reset();
523 }
524
525 public void reset() {
526 this.deletes.reset();
527 this.columns.reset();
528
529 stickyNextRow = false;
530 }
531
532
533
534
535
536 public KeyValue getStartKey() {
537 return this.startKey;
538 }
539
540
541
542
543
544 Filter getFilter() {
545 return this.filter;
546 }
547
548 public Cell getNextKeyHint(Cell kv) throws IOException {
549 if (filter == null) {
550 return null;
551 } else {
552 return filter.getNextCellHint(kv);
553 }
554 }
555
556 public KeyValue getKeyForNextColumn(KeyValue kv) {
557 ColumnCount nextColumn = columns.getColumnHint();
558 if (nextColumn == null) {
559 return KeyValue.createLastOnRow(
560 kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
561 kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
562 kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength());
563 } else {
564 return KeyValue.createFirstOnRow(
565 kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
566 kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
567 nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength());
568 }
569 }
570
571 public KeyValue getKeyForNextRow(KeyValue kv) {
572 return KeyValue.createLastOnRow(
573 kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(),
574 null, 0, 0,
575 null, 0, 0);
576 }
577
578
579 static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset,
580 int length, long ttl, byte type, boolean ignoreCount) throws IOException {
581 MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
582 if (matchCode == MatchCode.INCLUDE) {
583 return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
584 }
585 return matchCode;
586 }
587
588
589
590
591
592
593
594
595 public static enum MatchCode {
596
597
598
599 INCLUDE,
600
601
602
603
604 SKIP,
605
606
607
608
609 NEXT,
610
611
612
613
614 DONE,
615
616
617
618
619
620
621
622
623 SEEK_NEXT_ROW,
624
625
626
627 SEEK_NEXT_COL,
628
629
630
631
632 DONE_SCAN,
633
634
635
636
637 SEEK_NEXT_USING_HINT,
638
639
640
641
642 INCLUDE_AND_SEEK_NEXT_COL,
643
644
645
646
647 INCLUDE_AND_SEEK_NEXT_ROW,
648 }
649 }