1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.NavigableSet;
24
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.hbase.Cell;
27 import org.apache.hadoop.hbase.CellUtil;
28 import org.apache.hadoop.hbase.HConstants;
29 import org.apache.hadoop.hbase.KeepDeletedCells;
30 import org.apache.hadoop.hbase.KeyValue;
31 import org.apache.hadoop.hbase.KeyValueUtil;
32 import org.apache.hadoop.hbase.client.Scan;
33 import org.apache.hadoop.hbase.filter.Filter;
34 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
35 import org.apache.hadoop.hbase.io.TimeRange;
36 import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult;
37 import org.apache.hadoop.hbase.util.Bytes;
38 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
39
40 import com.google.common.base.Preconditions;
41
42
43
44
45 @InterfaceAudience.Private
46 public class ScanQueryMatcher {
47
48
49 private boolean stickyNextRow;
50 private final byte[] stopRow;
51
52 private final TimeRange tr;
53
54 private final Filter filter;
55
56
57 private final DeleteTracker deletes;
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73 private boolean retainDeletesInOutput;
74
75
76 private final KeepDeletedCells keepDeletedCells;
77
78 private final boolean seePastDeleteMarkers;
79
80
81
82 private final ColumnTracker columns;
83
84
85 private final Cell startKey;
86
87
88 private final KeyValue.KVComparator rowComparator;
89
90
91
92 byte [] row;
93 int rowOffset;
94 short rowLength;
95
96
97
98
99
100
101
102 private final long earliestPutTs;
103 private final long ttl;
104
105
106 private final long oldestUnexpiredTS;
107 private final long now;
108
109
110 protected long maxReadPointToTrackVersions;
111
112 private byte[] dropDeletesFromRow = null, dropDeletesToRow = null;
113
114
115
116
117
118
119
120 private boolean hasNullColumn = true;
121
122 private RegionCoprocessorHost regionCoprocessorHost= null;
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141 private final long timeToPurgeDeletes;
142
143 private final boolean isUserScan;
144
145 private final boolean isReversed;
146
147
148
149
150
151
152
153
154
155
156
157
158
159 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
160 ScanType scanType, long readPointToUse, long earliestPutTs, long oldestUnexpiredTS,
161 long now, RegionCoprocessorHost regionCoprocessorHost) throws IOException {
162 this.tr = scan.getTimeRange();
163 this.rowComparator = scanInfo.getComparator();
164 this.regionCoprocessorHost = regionCoprocessorHost;
165 this.deletes = instantiateDeleteTracker();
166 this.stopRow = scan.getStopRow();
167 this.startKey = KeyValueUtil.createFirstDeleteFamilyOnRow(scan.getStartRow(),
168 scanInfo.getFamily());
169 this.filter = scan.getFilter();
170 this.earliestPutTs = earliestPutTs;
171 this.oldestUnexpiredTS = oldestUnexpiredTS;
172 this.now = now;
173
174 this.maxReadPointToTrackVersions = readPointToUse;
175 this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes();
176 this.ttl = oldestUnexpiredTS;
177
178
179 this.isUserScan = scanType == ScanType.USER_SCAN;
180
181 this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE :
182 isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells();
183
184 this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw();
185
186 this.seePastDeleteMarkers =
187 scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan;
188
189 int maxVersions =
190 scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(),
191 scanInfo.getMaxVersions());
192
193
194 if (columns == null || columns.size() == 0) {
195
196 hasNullColumn = true;
197
198
199 this.columns = new ScanWildcardColumnTracker(
200 scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS);
201 } else {
202
203 hasNullColumn = (columns.first().length == 0);
204
205
206
207 byte[] attr = scan.getAttribute(Scan.HINT_LOOKAHEAD);
208 this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions,
209 oldestUnexpiredTS, attr == null ? 0 : Bytes.toInt(attr));
210 }
211 this.isReversed = scan.isReversed();
212 }
213
214 private DeleteTracker instantiateDeleteTracker() throws IOException {
215 DeleteTracker tracker = new ScanDeleteTracker();
216 if (regionCoprocessorHost != null) {
217 tracker = regionCoprocessorHost.postInstantiateDeleteTracker(tracker);
218 }
219 return tracker;
220 }
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235 public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet<byte[]> columns,
236 long readPointToUse, long earliestPutTs, long oldestUnexpiredTS, long now, byte[] dropDeletesFromRow,
237 byte[] dropDeletesToRow, RegionCoprocessorHost regionCoprocessorHost) throws IOException {
238 this(scan, scanInfo, columns, ScanType.COMPACT_RETAIN_DELETES, readPointToUse, earliestPutTs,
239 oldestUnexpiredTS, now, regionCoprocessorHost);
240 Preconditions.checkArgument((dropDeletesFromRow != null) && (dropDeletesToRow != null));
241 this.dropDeletesFromRow = dropDeletesFromRow;
242 this.dropDeletesToRow = dropDeletesToRow;
243 }
244
245
246
247
248 ScanQueryMatcher(Scan scan, ScanInfo scanInfo,
249 NavigableSet<byte[]> columns, long oldestUnexpiredTS, long now) throws IOException {
250 this(scan, scanInfo, columns, ScanType.USER_SCAN,
251 Long.MAX_VALUE,
252 HConstants.LATEST_TIMESTAMP, oldestUnexpiredTS, now, null);
253 }
254
255
256
257
258
259 public boolean hasNullColumnInQuery() {
260 return hasNullColumn;
261 }
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276 public MatchCode match(Cell cell) throws IOException {
277 if (filter != null && filter.filterAllRemaining()) {
278 return MatchCode.DONE_SCAN;
279 }
280 int ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength,
281 cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
282 if (!this.isReversed) {
283 if (ret <= -1) {
284 return MatchCode.DONE;
285 } else if (ret >= 1) {
286
287
288
289 return MatchCode.SEEK_NEXT_ROW;
290 }
291 } else {
292 if (ret <= -1) {
293 return MatchCode.SEEK_NEXT_ROW;
294 } else if (ret >= 1) {
295 return MatchCode.DONE;
296 }
297 }
298
299
300 if (this.stickyNextRow)
301 return MatchCode.SEEK_NEXT_ROW;
302
303 if (this.columns.done()) {
304 stickyNextRow = true;
305 return MatchCode.SEEK_NEXT_ROW;
306 }
307
308 int qualifierOffset = cell.getQualifierOffset();
309 int qualifierLength = cell.getQualifierLength();
310
311 long timestamp = cell.getTimestamp();
312
313 if (columns.isDone(timestamp)) {
314 return columns.getNextRowOrNextColumn(cell.getQualifierArray(), qualifierOffset,
315 qualifierLength);
316 }
317
318 if (HStore.isCellTTLExpired(cell, this.oldestUnexpiredTS, this.now)) {
319 return MatchCode.SKIP;
320 }
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335 byte typeByte = cell.getTypeByte();
336 long mvccVersion = cell.getMvccVersion();
337 if (CellUtil.isDelete(cell)) {
338 if (keepDeletedCells == KeepDeletedCells.FALSE
339 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) {
340
341
342
343
344
345
346 boolean includeDeleteMarker = seePastDeleteMarkers ?
347 tr.withinTimeRange(timestamp) :
348 tr.withinOrAfterTimeRange(timestamp);
349 if (includeDeleteMarker
350 && mvccVersion <= maxReadPointToTrackVersions) {
351 this.deletes.add(cell);
352 }
353
354 }
355
356 if ((!isUserScan)
357 && timeToPurgeDeletes > 0
358 && (EnvironmentEdgeManager.currentTime() - timestamp)
359 <= timeToPurgeDeletes) {
360 return MatchCode.INCLUDE;
361 } else if (retainDeletesInOutput || mvccVersion > maxReadPointToTrackVersions) {
362
363
364 if (!isUserScan) {
365
366
367 return MatchCode.INCLUDE;
368 }
369 } else if (keepDeletedCells == KeepDeletedCells.TRUE
370 || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) {
371 if (timestamp < earliestPutTs) {
372
373
374 return columns.getNextRowOrNextColumn(cell.getQualifierArray(),
375 qualifierOffset, qualifierLength);
376 }
377
378
379 } else {
380 return MatchCode.SKIP;
381 }
382
383
384 } else if (!this.deletes.isEmpty()) {
385 DeleteResult deleteResult = deletes.isDeleted(cell);
386 switch (deleteResult) {
387 case FAMILY_DELETED:
388 case COLUMN_DELETED:
389 return columns.getNextRowOrNextColumn(cell.getQualifierArray(),
390 qualifierOffset, qualifierLength);
391 case VERSION_DELETED:
392 case FAMILY_VERSION_DELETED:
393 return MatchCode.SKIP;
394 case NOT_DELETED:
395 break;
396 default:
397 throw new RuntimeException("UNEXPECTED");
398 }
399 }
400
401 int timestampComparison = tr.compare(timestamp);
402 if (timestampComparison >= 1) {
403 return MatchCode.SKIP;
404 } else if (timestampComparison <= -1) {
405 return columns.getNextRowOrNextColumn(cell.getQualifierArray(), qualifierOffset,
406 qualifierLength);
407 }
408
409
410 MatchCode colChecker = columns.checkColumn(cell.getQualifierArray(),
411 qualifierOffset, qualifierLength, typeByte);
412 if (colChecker == MatchCode.INCLUDE) {
413 ReturnCode filterResponse = ReturnCode.SKIP;
414
415 if (filter != null) {
416
417 filterResponse = filter.filterKeyValue(cell);
418 switch (filterResponse) {
419 case SKIP:
420 return MatchCode.SKIP;
421 case NEXT_COL:
422 return columns.getNextRowOrNextColumn(cell.getQualifierArray(),
423 qualifierOffset, qualifierLength);
424 case NEXT_ROW:
425 stickyNextRow = true;
426 return MatchCode.SEEK_NEXT_ROW;
427 case SEEK_NEXT_USING_HINT:
428 return MatchCode.SEEK_NEXT_USING_HINT;
429 default:
430
431 break;
432 }
433 }
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453 colChecker =
454 columns.checkVersions(cell.getQualifierArray(), qualifierOffset,
455 qualifierLength, timestamp, typeByte,
456 mvccVersion > maxReadPointToTrackVersions);
457
458 stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow;
459 return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL &&
460 colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL
461 : colChecker;
462 }
463 stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true
464 : stickyNextRow;
465 return colChecker;
466 }
467
468
469
470
471 private void checkPartialDropDeleteRange(byte [] row, int offset, short length) {
472
473
474
475
476 if ((dropDeletesFromRow != null)
477 && ((dropDeletesFromRow == HConstants.EMPTY_START_ROW)
478 || (Bytes.compareTo(row, offset, length,
479 dropDeletesFromRow, 0, dropDeletesFromRow.length) >= 0))) {
480 retainDeletesInOutput = false;
481 dropDeletesFromRow = null;
482 }
483
484
485
486 if ((dropDeletesFromRow == null)
487 && (dropDeletesToRow != null) && (dropDeletesToRow != HConstants.EMPTY_END_ROW)
488 && (Bytes.compareTo(row, offset, length,
489 dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) {
490 retainDeletesInOutput = true;
491 dropDeletesToRow = null;
492 }
493 }
494
495 public boolean moreRowsMayExistAfter(Cell kv) {
496 if (this.isReversed) {
497 if (rowComparator.compareRows(kv.getRowArray(), kv.getRowOffset(),
498 kv.getRowLength(), stopRow, 0, stopRow.length) <= 0) {
499 return false;
500 } else {
501 return true;
502 }
503 }
504 if (!Bytes.equals(stopRow , HConstants.EMPTY_END_ROW) &&
505 rowComparator.compareRows(kv.getRowArray(),kv.getRowOffset(),
506 kv.getRowLength(), stopRow, 0, stopRow.length) >= 0) {
507
508
509 return false;
510 } else {
511 return true;
512 }
513 }
514
515
516
517
518
519 public void setRow(byte [] row, int offset, short length) {
520 checkPartialDropDeleteRange(row, offset, length);
521 this.row = row;
522 this.rowOffset = offset;
523 this.rowLength = length;
524 reset();
525 }
526
527 public void reset() {
528 this.deletes.reset();
529 this.columns.reset();
530
531 stickyNextRow = false;
532 }
533
534
535
536
537
538 public Cell getStartKey() {
539 return this.startKey;
540 }
541
542
543
544
545
546 Filter getFilter() {
547 return this.filter;
548 }
549
550 public Cell getNextKeyHint(Cell kv) throws IOException {
551 if (filter == null) {
552 return null;
553 } else {
554 return filter.getNextCellHint(kv);
555 }
556 }
557
558 public Cell getKeyForNextColumn(Cell kv) {
559 ColumnCount nextColumn = columns.getColumnHint();
560 if (nextColumn == null) {
561 return KeyValueUtil.createLastOnRow(
562 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
563 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
564 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength());
565 } else {
566 return KeyValueUtil.createFirstOnRow(
567 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
568 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
569 nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength());
570 }
571 }
572
573 public Cell getKeyForNextRow(Cell kv) {
574 return KeyValueUtil.createLastOnRow(
575 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
576 null, 0, 0,
577 null, 0, 0);
578 }
579
580
581 static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset,
582 int length, long ttl, byte type, boolean ignoreCount) throws IOException {
583 MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
584 if (matchCode == MatchCode.INCLUDE) {
585 return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
586 }
587 return matchCode;
588 }
589
590
591
592
593
594
595
596
597 public static enum MatchCode {
598
599
600
601 INCLUDE,
602
603
604
605
606 SKIP,
607
608
609
610
611 NEXT,
612
613
614
615
616 DONE,
617
618
619
620
621
622
623
624
625 SEEK_NEXT_ROW,
626
627
628
629 SEEK_NEXT_COL,
630
631
632
633
634 DONE_SCAN,
635
636
637
638
639 SEEK_NEXT_USING_HINT,
640
641
642
643
644 INCLUDE_AND_SEEK_NEXT_COL,
645
646
647
648
649 INCLUDE_AND_SEEK_NEXT_ROW,
650 }
651 }