View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.regionserver;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableSet;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.DoNotRetryIOException;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.client.Scan;
34  import org.apache.hadoop.hbase.filter.Filter;
35  import org.apache.hadoop.hbase.regionserver.Store.ScanInfo;
36  import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage;
37  import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
40  
41  /**
42   * Scanner scans both the memstore and the HStore. Coalesce KeyValue stream
43   * into List<KeyValue> for a single row.
44   */
45  public class StoreScanner extends NonLazyKeyValueScanner
46      implements KeyValueScanner, InternalScanner, ChangedReadersObserver {
47    static final Log LOG = LogFactory.getLog(StoreScanner.class);
48    private Store store;
49    private ScanQueryMatcher matcher;
50    private KeyValueHeap heap;
51    private boolean cacheBlocks;
52  
53  
54    private String metricNamePrefix;
55    // Used to indicate that the scanner has closed (see HBASE-1107)
56    // Doesnt need to be volatile because it's always accessed via synchronized methods
57    private boolean closing = false;
58    private final boolean isGet;
59    private final boolean explicitColumnQuery;
60    private final boolean useRowColBloom;
61    private final Scan scan;
62    private final NavigableSet<byte[]> columns;
63    private final long oldestUnexpiredTS;
64    private final int minVersions;
65  
66    /** We don't ever expect to change this, the constant is just for clarity. */
67    static final boolean LAZY_SEEK_ENABLED_BY_DEFAULT = true;
68  
69    /** Used during unit testing to ensure that lazy seek does save seek ops */
70    private static boolean lazySeekEnabledGlobally =
71        LAZY_SEEK_ENABLED_BY_DEFAULT;
72  
73    // if heap == null and lastTop != null, you need to reseek given the key below
74    private KeyValue lastTop = null;
75  
76    /** An internal constructor. */
77    private StoreScanner(Store store, boolean cacheBlocks, Scan scan,
78        final NavigableSet<byte[]> columns, long ttl, int minVersions) {
79      this.store = store;
80      this.cacheBlocks = cacheBlocks;
81      isGet = scan.isGetScan();
82      int numCol = columns == null ? 0 : columns.size();
83      explicitColumnQuery = numCol > 0;
84      this.scan = scan;
85      this.columns = columns;
86      oldestUnexpiredTS = EnvironmentEdgeManager.currentTimeMillis() - ttl;
87      this.minVersions = minVersions;
88  
89      // We look up row-column Bloom filters for multi-column queries as part of
90      // the seek operation. However, we also look the row-column Bloom filter
91      // for multi-row (non-"get") scans because this is not done in
92      // StoreFile.passesBloomFilter(Scan, SortedSet<byte[]>).
93      useRowColBloom = numCol > 1 || (!isGet && numCol == 1);
94    }
95  
96    /**
97     * Opens a scanner across memstore, snapshot, and all StoreFiles. Assumes we
98     * are not in a compaction.
99     *
100    * @param store who we scan
101    * @param scan the spec
102    * @param columns which columns we are scanning
103    * @throws IOException
104    */
105   public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, final NavigableSet<byte[]> columns)
106                               throws IOException {
107     this(store, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(),
108         scanInfo.getMinVersions());
109     initializeMetricNames();
110     if (columns != null && scan.isRaw()) {
111       throw new DoNotRetryIOException(
112           "Cannot specify any column for a raw scan");
113     }
114     matcher = new ScanQueryMatcher(scan, scanInfo, columns,
115         ScanType.USER_SCAN, Long.MAX_VALUE, HConstants.LATEST_TIMESTAMP,
116         oldestUnexpiredTS);
117 
118     // Pass columns to try to filter out unnecessary StoreFiles.
119     List<KeyValueScanner> scanners = getScannersNoCompaction();
120 
121     // Seek all scanners to the start of the Row (or if the exact matching row
122     // key does not exist, then to the start of the next matching Row).
123     // Always check bloom filter to optimize the top row seek for delete
124     // family marker.
125     if (explicitColumnQuery && lazySeekEnabledGlobally) {
126       for (KeyValueScanner scanner : scanners) {
127         scanner.requestSeek(matcher.getStartKey(), false, true);
128       }
129     } else {
130       for (KeyValueScanner scanner : scanners) {
131         scanner.seek(matcher.getStartKey());
132       }
133     }
134 
135     // Combine all seeked scanners with a heap
136     heap = new KeyValueHeap(scanners, store.comparator);
137 
138     this.store.addChangedReaderObserver(this);
139   }
140 
141   /**
142    * Used for major compactions.<p>
143    *
144    * Opens a scanner across specified StoreFiles.
145    * @param store who we scan
146    * @param scan the spec
147    * @param scanners ancillary scanners
148    * @param smallestReadPoint the readPoint that we should use for tracking
149    *          versions
150    */
151   public StoreScanner(Store store, ScanInfo scanInfo, Scan scan,
152       List<? extends KeyValueScanner> scanners, ScanType scanType,
153       long smallestReadPoint, long earliestPutTs) throws IOException {
154     this(store, false, scan, null, scanInfo.getTtl(),
155         scanInfo.getMinVersions());
156     initializeMetricNames();
157     matcher = new ScanQueryMatcher(scan, scanInfo, null, scanType,
158         smallestReadPoint, earliestPutTs, oldestUnexpiredTS);
159 
160     // Filter the list of scanners using Bloom filters, time range, TTL, etc.
161     scanners = selectScannersFrom(scanners);
162 
163     // Seek all scanners to the initial key
164     for(KeyValueScanner scanner : scanners) {
165       scanner.seek(matcher.getStartKey());
166     }
167 
168     // Combine all seeked scanners with a heap
169     heap = new KeyValueHeap(scanners, store.comparator);
170   }
171 
172   /** Constructor for testing. */
173   StoreScanner(final Scan scan, Store.ScanInfo scanInfo,
174       ScanType scanType, final NavigableSet<byte[]> columns,
175       final List<KeyValueScanner> scanners) throws IOException {
176     this(scan, scanInfo, scanType, columns, scanners,
177         HConstants.LATEST_TIMESTAMP);
178   }
179 
180   // Constructor for testing.
181   StoreScanner(final Scan scan, Store.ScanInfo scanInfo,
182       ScanType scanType, final NavigableSet<byte[]> columns,
183       final List<KeyValueScanner> scanners, long earliestPutTs)
184           throws IOException {
185     this(null, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(),
186         scanInfo.getMinVersions());
187     this.initializeMetricNames();
188     this.matcher = new ScanQueryMatcher(scan, scanInfo, columns, scanType,
189         Long.MAX_VALUE, earliestPutTs, oldestUnexpiredTS);
190 
191     // Seek all scanners to the initial key
192     for (KeyValueScanner scanner : scanners) {
193       scanner.seek(matcher.getStartKey());
194     }
195     heap = new KeyValueHeap(scanners, scanInfo.getComparator());
196   }
197 
198   /**
199    * Method used internally to initialize metric names throughout the
200    * constructors.
201    *
202    * To be called after the store variable has been initialized!
203    */
204   private void initializeMetricNames() {
205     String tableName = SchemaMetrics.UNKNOWN;
206     String family = SchemaMetrics.UNKNOWN;
207     if (store != null) {
208       tableName = store.getTableName();
209       family = Bytes.toString(store.getFamily().getName());
210     }
211     this.metricNamePrefix =
212         SchemaMetrics.generateSchemaMetricsPrefix(tableName, family);
213   }
214 
215   /**
216    * Get a filtered list of scanners. Assumes we are not in a compaction.
217    * @return list of scanners to seek
218    */
219   private List<KeyValueScanner> getScannersNoCompaction() throws IOException {
220     final boolean isCompaction = false;
221     return selectScannersFrom(store.getScanners(cacheBlocks, isGet,
222         isCompaction, matcher));
223   }
224 
225   /**
226    * Filters the given list of scanners using Bloom filter, time range, and
227    * TTL.
228    */
229   private List<KeyValueScanner> selectScannersFrom(
230       final List<? extends KeyValueScanner> allScanners) {
231     boolean memOnly;
232     boolean filesOnly;
233     if (scan instanceof InternalScan) {
234       InternalScan iscan = (InternalScan)scan;
235       memOnly = iscan.isCheckOnlyMemStore();
236       filesOnly = iscan.isCheckOnlyStoreFiles();
237     } else {
238       memOnly = false;
239       filesOnly = false;
240     }
241 
242     List<KeyValueScanner> scanners =
243         new ArrayList<KeyValueScanner>(allScanners.size());
244 
245     // We can only exclude store files based on TTL if minVersions is set to 0.
246     // Otherwise, we might have to return KVs that have technically expired.
247     long expiredTimestampCutoff = minVersions == 0 ? oldestUnexpiredTS :
248         Long.MIN_VALUE;
249 
250     // include only those scan files which pass all filters
251     for (KeyValueScanner kvs : allScanners) {
252       boolean isFile = kvs.isFileScanner();
253       if ((!isFile && filesOnly) || (isFile && memOnly)) {
254         continue;
255       }
256 
257       if (kvs.shouldUseScanner(scan, columns, expiredTimestampCutoff)) {
258         scanners.add(kvs);
259       }
260     }
261     return scanners;
262   }
263 
264   @Override
265   public synchronized KeyValue peek() {
266     if (this.heap == null) {
267       return this.lastTop;
268     }
269     return this.heap.peek();
270   }
271 
272   @Override
273   public KeyValue next() {
274     // throw runtime exception perhaps?
275     throw new RuntimeException("Never call StoreScanner.next()");
276   }
277 
278   @Override
279   public synchronized void close() {
280     if (this.closing) return;
281     this.closing = true;
282     // under test, we dont have a this.store
283     if (this.store != null)
284       this.store.deleteChangedReaderObserver(this);
285     if (this.heap != null)
286       this.heap.close();
287     this.heap = null; // CLOSED!
288     this.lastTop = null; // If both are null, we are closed.
289   }
290 
291   @Override
292   public synchronized boolean seek(KeyValue key) throws IOException {
293     if (this.heap == null) {
294 
295       List<KeyValueScanner> scanners = getScannersNoCompaction();
296 
297       heap = new KeyValueHeap(scanners, store.comparator);
298     }
299 
300     return this.heap.seek(key);
301   }
302 
303   /**
304    * Get the next row of values from this Store.
305    * @param outResult
306    * @param limit
307    * @return true if there are more rows, false if scanner is done
308    */
309   @Override
310   public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOException {
311     return next(outResult, limit, null);
312   }
313 
314   /**
315    * Get the next row of values from this Store.
316    * @param outResult
317    * @param limit
318    * @return true if there are more rows, false if scanner is done
319    */
320   @Override
321   public synchronized boolean next(List<KeyValue> outResult, int limit,
322       String metric) throws IOException {
323 
324     if (checkReseek()) {
325       return true;
326     }
327 
328     // if the heap was left null, then the scanners had previously run out anyways, close and
329     // return.
330     if (this.heap == null) {
331       close();
332       return false;
333     }
334 
335     KeyValue peeked = this.heap.peek();
336     if (peeked == null) {
337       close();
338       return false;
339     }
340 
341     // only call setRow if the row changes; avoids confusing the query matcher
342     // if scanning intra-row
343     byte[] row = peeked.getBuffer();
344     int offset = peeked.getRowOffset();
345     short length = peeked.getRowLength();
346     if (limit < 0 || matcher.row == null || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) {
347       matcher.setRow(row, offset, length);
348     }
349 
350     KeyValue kv;
351     KeyValue prevKV = null;
352 
353     // Only do a sanity-check if store and comparator are available.
354     KeyValue.KVComparator comparator =
355         store != null ? store.getComparator() : null;
356 
357     long cumulativeMetric = 0;
358     int count = 0;
359     try {
360       LOOP: while((kv = this.heap.peek()) != null) {
361         // Check that the heap gives us KVs in an increasing order.
362         assert prevKV == null || comparator == null || comparator.compare(prevKV, kv) <= 0 :
363           "Key " + prevKV + " followed by a " + "smaller key " + kv + " in cf " + store;
364         prevKV = kv;
365         ScanQueryMatcher.MatchCode qcode = matcher.match(kv);
366         switch(qcode) {
367           case INCLUDE:
368           case INCLUDE_AND_SEEK_NEXT_ROW:
369           case INCLUDE_AND_SEEK_NEXT_COL:
370 
371             Filter f = matcher.getFilter();
372             outResult.add(f == null ? kv : f.transform(kv));
373             count++;
374 
375             if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW) {
376               if (!matcher.moreRowsMayExistAfter(kv)) {
377                 return false;
378               }
379               reseek(matcher.getKeyForNextRow(kv));
380             } else if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL) {
381               reseek(matcher.getKeyForNextColumn(kv));
382             } else {
383               this.heap.next();
384             }
385 
386             cumulativeMetric += kv.getLength();
387             if (limit > 0 && (count == limit)) {
388               break LOOP;
389             }
390             continue;
391 
392           case DONE:
393             return true;
394 
395           case DONE_SCAN:
396             close();
397 
398             return false;
399 
400           case SEEK_NEXT_ROW:
401             // This is just a relatively simple end of scan fix, to short-cut end
402             // us if there is an endKey in the scan.
403             if (!matcher.moreRowsMayExistAfter(kv)) {
404               return false;
405             }
406 
407             reseek(matcher.getKeyForNextRow(kv));
408             break;
409 
410           case SEEK_NEXT_COL:
411             reseek(matcher.getKeyForNextColumn(kv));
412             break;
413 
414           case SKIP:
415             this.heap.next();
416             break;
417 
418           case SEEK_NEXT_USING_HINT:
419             KeyValue nextKV = matcher.getNextKeyHint(kv);
420             if (nextKV != null) {
421               reseek(nextKV);
422             } else {
423               heap.next();
424             }
425             break;
426 
427           default:
428             throw new RuntimeException("UNEXPECTED");
429         }
430       }
431     } finally {
432       if (cumulativeMetric > 0 && metric != null) {
433         RegionMetricsStorage.incrNumericMetric(this.metricNamePrefix + metric,
434             cumulativeMetric);
435       }
436     }
437 
438     if (count > 0) {
439       return true;
440     }
441 
442     // No more keys
443     close();
444     return false;
445   }
446 
447   @Override
448   public synchronized boolean next(List<KeyValue> outResult) throws IOException {
449     return next(outResult, -1, null);
450   }
451 
452   @Override
453   public synchronized boolean next(List<KeyValue> outResult, String metric)
454       throws IOException {
455     return next(outResult, -1, metric);
456   }
457 
458   // Implementation of ChangedReadersObserver
459   @Override
460   public synchronized void updateReaders() throws IOException {
461     if (this.closing) return;
462 
463     // All public synchronized API calls will call 'checkReseek' which will cause
464     // the scanner stack to reseek if this.heap==null && this.lastTop != null.
465     // But if two calls to updateReaders() happen without a 'next' or 'peek' then we
466     // will end up calling this.peek() which would cause a reseek in the middle of a updateReaders
467     // which is NOT what we want, not to mention could cause an NPE. So we early out here.
468     if (this.heap == null) return;
469 
470     // this could be null.
471     this.lastTop = this.peek();
472 
473     //DebugPrint.println("SS updateReaders, topKey = " + lastTop);
474 
475     // close scanners to old obsolete Store files
476     this.heap.close(); // bubble thru and close all scanners.
477     this.heap = null; // the re-seeks could be slow (access HDFS) free up memory ASAP
478 
479     // Let the next() call handle re-creating and seeking
480   }
481 
482   /**
483    * @return true if top of heap has changed (and KeyValueHeap has to try the
484    *         next KV)
485    * @throws IOException
486    */
487   private boolean checkReseek() throws IOException {
488     if (this.heap == null && this.lastTop != null) {
489       resetScannerStack(this.lastTop);
490       if (this.heap.peek() == null
491           || store.comparator.compareRows(this.lastTop, this.heap.peek()) != 0) {
492         LOG.debug("Storescanner.peek() is changed where before = "
493             + this.lastTop.toString() + ",and after = " + this.heap.peek());
494         this.lastTop = null;
495         return true;
496       }
497       this.lastTop = null; // gone!
498     }
499     // else dont need to reseek
500     return false;
501   }
502 
503   private void resetScannerStack(KeyValue lastTopKey) throws IOException {
504     if (heap != null) {
505       throw new RuntimeException("StoreScanner.reseek run on an existing heap!");
506     }
507 
508     /* When we have the scan object, should we not pass it to getScanners()
509      * to get a limited set of scanners? We did so in the constructor and we
510      * could have done it now by storing the scan object from the constructor */
511     List<KeyValueScanner> scanners = getScannersNoCompaction();
512 
513     for(KeyValueScanner scanner : scanners) {
514       scanner.seek(lastTopKey);
515     }
516 
517     // Combine all seeked scanners with a heap
518     heap = new KeyValueHeap(scanners, store.comparator);
519 
520     // Reset the state of the Query Matcher and set to top row.
521     // Only reset and call setRow if the row changes; avoids confusing the
522     // query matcher if scanning intra-row.
523     KeyValue kv = heap.peek();
524     if (kv == null) {
525       kv = lastTopKey;
526     }
527     byte[] row = kv.getBuffer();
528     int offset = kv.getRowOffset();
529     short length = kv.getRowLength();
530     if ((matcher.row == null) || !Bytes.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) {
531       matcher.reset();
532       matcher.setRow(row, offset, length);
533     }
534   }
535 
536   @Override
537   public synchronized boolean reseek(KeyValue kv) throws IOException {
538     //Heap will not be null, if this is called from next() which.
539     //If called from RegionScanner.reseek(...) make sure the scanner
540     //stack is reset if needed.
541     checkReseek();
542     if (explicitColumnQuery && lazySeekEnabledGlobally) {
543       return heap.requestSeek(kv, true, useRowColBloom);
544     } else {
545       return heap.reseek(kv);
546     }
547   }
548 
549   @Override
550   public long getSequenceID() {
551     return 0;
552   }
553 
554   /**
555    * Used in testing.
556    * @return all scanners in no particular order
557    */
558   List<KeyValueScanner> getAllScannersForTesting() {
559     List<KeyValueScanner> allScanners = new ArrayList<KeyValueScanner>();
560     KeyValueScanner current = heap.getCurrentForTesting();
561     if (current != null)
562       allScanners.add(current);
563     for (KeyValueScanner scanner : heap.getHeap())
564       allScanners.add(scanner);
565     return allScanners;
566   }
567 
568   static void enableLazySeekGlobally(boolean enable) {
569     lazySeekEnabledGlobally = enable;
570   }
571 }
572