1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.FileNotFoundException;
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.List;
27 import java.util.SortedSet;
28 import java.util.concurrent.atomic.AtomicLong;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.KeyValue;
35 import org.apache.hadoop.hbase.client.Scan;
36 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
37 import org.apache.hadoop.hbase.regionserver.StoreFile.Reader;
38
39
40
41
42
43 @InterfaceAudience.LimitedPrivate("Coprocessor")
44 public class StoreFileScanner implements KeyValueScanner {
45 static final Log LOG = LogFactory.getLog(HStore.class);
46
47
48 private final StoreFile.Reader reader;
49 private final HFileScanner hfs;
50 private KeyValue cur = null;
51
52 private boolean realSeekDone;
53 private boolean delayedReseek;
54 private KeyValue delayedSeekKV;
55
56 private boolean enforceMVCC = false;
57 private boolean hasMVCCInfo = false;
58
59
60 private boolean stopSkippingKVsIfNextRow = false;
61
62 private static AtomicLong seekCount;
63
64 private ScanQueryMatcher matcher;
65
66 private long readPt;
67
68
69
70
71
72 public StoreFileScanner(StoreFile.Reader reader, HFileScanner hfs, boolean useMVCC,
73 boolean hasMVCC, long readPt) {
74 this.readPt = readPt;
75 this.reader = reader;
76 this.hfs = hfs;
77 this.enforceMVCC = useMVCC;
78 this.hasMVCCInfo = hasMVCC;
79 }
80
81
82
83
84
85 public static List<StoreFileScanner> getScannersForStoreFiles(
86 Collection<StoreFile> files,
87 boolean cacheBlocks,
88 boolean usePread, long readPt) throws IOException {
89 return getScannersForStoreFiles(files, cacheBlocks,
90 usePread, false, readPt);
91 }
92
93
94
95
96 public static List<StoreFileScanner> getScannersForStoreFiles(
97 Collection<StoreFile> files, boolean cacheBlocks, boolean usePread,
98 boolean isCompaction, long readPt) throws IOException {
99 return getScannersForStoreFiles(files, cacheBlocks, usePread, isCompaction,
100 null, readPt);
101 }
102
103
104
105
106
107
108 public static List<StoreFileScanner> getScannersForStoreFiles(
109 Collection<StoreFile> files, boolean cacheBlocks, boolean usePread,
110 boolean isCompaction, ScanQueryMatcher matcher, long readPt) throws IOException {
111 List<StoreFileScanner> scanners = new ArrayList<StoreFileScanner>(
112 files.size());
113 for (StoreFile file : files) {
114 StoreFile.Reader r = file.createReader();
115 StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, usePread,
116 isCompaction, readPt);
117 scanner.setScanQueryMatcher(matcher);
118 scanners.add(scanner);
119 }
120 return scanners;
121 }
122
123 public String toString() {
124 return "StoreFileScanner[" + hfs.toString() + ", cur=" + cur + "]";
125 }
126
127 public KeyValue peek() {
128 return cur;
129 }
130
131 public KeyValue next() throws IOException {
132 KeyValue retKey = cur;
133
134 try {
135
136 if (cur != null) {
137 hfs.next();
138 cur = hfs.getKeyValue();
139 if (hasMVCCInfo)
140 skipKVsNewerThanReadpoint();
141 }
142 } catch (FileNotFoundException e) {
143 throw e;
144 } catch(IOException e) {
145 throw new IOException("Could not iterate " + this, e);
146 }
147 return retKey;
148 }
149
150 public boolean seek(KeyValue key) throws IOException {
151 if (seekCount != null) seekCount.incrementAndGet();
152
153 try {
154 try {
155 if(!seekAtOrAfter(hfs, key)) {
156 close();
157 return false;
158 }
159
160 cur = hfs.getKeyValue();
161
162 return !hasMVCCInfo ? true : skipKVsNewerThanReadpoint();
163 } finally {
164 realSeekDone = true;
165 }
166 } catch (FileNotFoundException e) {
167 throw e;
168 } catch (IOException ioe) {
169 throw new IOException("Could not seek " + this + " to key " + key, ioe);
170 }
171 }
172
173 public boolean reseek(KeyValue key) throws IOException {
174 if (seekCount != null) seekCount.incrementAndGet();
175
176 try {
177 try {
178 if (!reseekAtOrAfter(hfs, key)) {
179 close();
180 return false;
181 }
182 cur = hfs.getKeyValue();
183
184 return !hasMVCCInfo ? true : skipKVsNewerThanReadpoint();
185 } finally {
186 realSeekDone = true;
187 }
188 } catch (FileNotFoundException e) {
189 throw e;
190 } catch (IOException ioe) {
191 throw new IOException("Could not reseek " + this + " to key " + key,
192 ioe);
193 }
194 }
195
196 protected boolean skipKVsNewerThanReadpoint() throws IOException {
197
198
199 KeyValue startKV = cur;
200 while(enforceMVCC
201 && cur != null
202 && (cur.getMvccVersion() > readPt)) {
203 hfs.next();
204 cur = hfs.getKeyValue();
205 if (this.stopSkippingKVsIfNextRow
206 && getComparator().compareRows(cur.getBuffer(), cur.getRowOffset(),
207 cur.getRowLength(), startKV.getBuffer(), startKV.getRowOffset(),
208 startKV.getRowLength()) > 0) {
209 return false;
210 }
211 }
212
213 if (cur == null) {
214 close();
215 return false;
216 }
217
218
219
220
221
222
223
224 if (cur.getMvccVersion() <= readPt) {
225 cur.setMvccVersion(0);
226 }
227 return true;
228 }
229
230 public void close() {
231
232 cur = null;
233 }
234
235
236
237
238
239
240
241
242 public static boolean seekAtOrAfter(HFileScanner s, KeyValue k)
243 throws IOException {
244 int result = s.seekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength());
245 if(result < 0) {
246 if (result == HConstants.INDEX_KEY_MAGIC) {
247
248 return true;
249 }
250
251 return s.seekTo();
252 } else if(result > 0) {
253
254
255 return s.next();
256 }
257
258 return true;
259 }
260
261 static boolean reseekAtOrAfter(HFileScanner s, KeyValue k)
262 throws IOException {
263
264 int result = s.reseekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength());
265 if (result <= 0) {
266 if (result == HConstants.INDEX_KEY_MAGIC) {
267
268 return true;
269 }
270
271
272
273 if (!s.isSeeked()) {
274 return s.seekTo();
275 }
276 return true;
277 }
278
279
280 return s.next();
281 }
282
283 @Override
284 public long getSequenceID() {
285 return reader.getSequenceID();
286 }
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302 @Override
303 public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom)
304 throws IOException {
305 if (kv.getFamilyLength() == 0) {
306 useBloom = false;
307 }
308
309 boolean haveToSeek = true;
310 if (useBloom) {
311
312 if (reader.getBloomFilterType() == BloomType.ROWCOL) {
313 haveToSeek = reader.passesGeneralBloomFilter(kv.getBuffer(),
314 kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(),
315 kv.getQualifierOffset(), kv.getQualifierLength());
316 } else if (this.matcher != null && !matcher.hasNullColumnInQuery() &&
317 (kv.isDeleteFamily() || kv.isDeleteFamilyVersion())) {
318
319
320 haveToSeek = reader.passesDeleteFamilyBloomFilter(kv.getBuffer(),
321 kv.getRowOffset(), kv.getRowLength());
322 }
323 }
324
325 delayedReseek = forward;
326 delayedSeekKV = kv;
327
328 if (haveToSeek) {
329
330
331 realSeekDone = false;
332 long maxTimestampInFile = reader.getMaxTimestamp();
333 long seekTimestamp = kv.getTimestamp();
334 if (seekTimestamp > maxTimestampInFile) {
335
336
337
338
339
340
341 cur = kv.createFirstOnRowColTS(maxTimestampInFile);
342 } else {
343
344
345
346
347 enforceSeek();
348 }
349 return cur != null;
350 }
351
352
353
354
355
356
357
358
359 cur = kv.createLastOnRowCol();
360
361 realSeekDone = true;
362 return true;
363 }
364
365 Reader getReaderForTesting() {
366 return reader;
367 }
368
369 KeyValue.KVComparator getComparator() {
370 return reader.getComparator();
371 }
372
373 @Override
374 public boolean realSeekDone() {
375 return realSeekDone;
376 }
377
378 @Override
379 public void enforceSeek() throws IOException {
380 if (realSeekDone)
381 return;
382
383 if (delayedReseek) {
384 reseek(delayedSeekKV);
385 } else {
386 seek(delayedSeekKV);
387 }
388 }
389
390 public void setScanQueryMatcher(ScanQueryMatcher matcher) {
391 this.matcher = matcher;
392 }
393
394 @Override
395 public boolean isFileScanner() {
396 return true;
397 }
398
399
400
401 static final long getSeekCount() {
402 return seekCount.get();
403 }
404 static final void instrument() {
405 seekCount = new AtomicLong();
406 }
407
408 @Override
409 public boolean shouldUseScanner(Scan scan, SortedSet<byte[]> columns, long oldestUnexpiredTS) {
410 return reader.passesTimerangeFilter(scan, oldestUnexpiredTS)
411 && reader.passesKeyRangeFilter(scan) && reader.passesBloomFilter(scan, columns);
412 }
413
414 @Override
415 public boolean seekToPreviousRow(KeyValue key) throws IOException {
416 try {
417 try {
418 KeyValue seekKey = KeyValue.createFirstOnRow(key.getRow());
419 if (seekCount != null) seekCount.incrementAndGet();
420 if (!hfs.seekBefore(seekKey.getBuffer(), seekKey.getKeyOffset(),
421 seekKey.getKeyLength())) {
422 close();
423 return false;
424 }
425 KeyValue firstKeyOfPreviousRow = KeyValue.createFirstOnRow(hfs
426 .getKeyValue().getRow());
427
428 if (seekCount != null) seekCount.incrementAndGet();
429 if (!seekAtOrAfter(hfs, firstKeyOfPreviousRow)) {
430 close();
431 return false;
432 }
433
434 cur = hfs.getKeyValue();
435 this.stopSkippingKVsIfNextRow = true;
436 boolean resultOfSkipKVs;
437 try {
438 resultOfSkipKVs = skipKVsNewerThanReadpoint();
439 } finally {
440 this.stopSkippingKVsIfNextRow = false;
441 }
442 if (!resultOfSkipKVs
443 || getComparator().compareRows(cur.getBuffer(), cur.getRowOffset(),
444 cur.getRowLength(), firstKeyOfPreviousRow.getBuffer(),
445 firstKeyOfPreviousRow.getRowOffset(),
446 firstKeyOfPreviousRow.getRowLength()) > 0) {
447 return seekToPreviousRow(firstKeyOfPreviousRow);
448 }
449
450 return true;
451 } finally {
452 realSeekDone = true;
453 }
454 } catch (IOException ioe) {
455 throw new IOException("Could not seekToPreviousRow " + this + " to key "
456 + key, ioe);
457 }
458 }
459
460 @Override
461 public boolean seekToLastRow() throws IOException {
462 byte[] lastRow = reader.getLastRowKey();
463 if (lastRow == null) {
464 return false;
465 }
466 KeyValue seekKey = KeyValue.createFirstOnRow(lastRow);
467 if (seek(seekKey)) {
468 return true;
469 } else {
470 return seekToPreviousRow(seekKey);
471 }
472 }
473
474 @Override
475 public boolean backwardSeek(KeyValue key) throws IOException {
476 seek(key);
477 if (cur == null
478 || getComparator().compareRows(cur.getRowArray(), cur.getRowOffset(),
479 cur.getRowLength(), key.getRowArray(), key.getRowOffset(),
480 key.getRowLength()) > 0) {
481 return seekToPreviousRow(key);
482 }
483 return true;
484 }
485
486 @Override
487 public byte[] getNextIndexedKey() {
488 return hfs.getNextIndexedKey();
489 }
490 }