View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.DataInput;
22  import java.io.IOException;
23  import java.net.InetSocketAddress;
24  import java.nio.ByteBuffer;
25  import java.util.Arrays;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.Comparator;
29  import java.util.Map;
30  import java.util.SortedSet;
31  import java.util.UUID;
32  import java.util.concurrent.atomic.AtomicBoolean;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
41  import org.apache.hadoop.hbase.KeyValue;
42  import org.apache.hadoop.hbase.KeyValue.KVComparator;
43  import org.apache.hadoop.hbase.classification.InterfaceAudience;
44  import org.apache.hadoop.hbase.client.Scan;
45  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
46  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
47  import org.apache.hadoop.hbase.io.hfile.BlockType;
48  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
49  import org.apache.hadoop.hbase.io.hfile.HFile;
50  import org.apache.hadoop.hbase.io.hfile.HFileContext;
51  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
52  import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
53  import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
54  import org.apache.hadoop.hbase.util.BloomFilter;
55  import org.apache.hadoop.hbase.util.BloomFilterFactory;
56  import org.apache.hadoop.hbase.util.BloomFilterWriter;
57  import org.apache.hadoop.hbase.util.Bytes;
58  import org.apache.hadoop.hbase.util.ChecksumType;
59  import org.apache.hadoop.hbase.util.Writables;
60  import org.apache.hadoop.io.WritableUtils;
61  
62  import com.google.common.base.Function;
63  import com.google.common.base.Preconditions;
64  import com.google.common.collect.ImmutableList;
65  import com.google.common.collect.Ordering;
66  
67  /**
68   * A Store data file.  Stores usually have one or more of these files.  They
69   * are produced by flushing the memstore to disk.  To
70   * create, instantiate a writer using {@link StoreFile.WriterBuilder}
71   * and append data. Be sure to add any metadata before calling close on the
72   * Writer (Use the appendMetadata convenience methods). On close, a StoreFile
73   * is sitting in the Filesystem.  To refer to it, create a StoreFile instance
74   * passing filesystem and path.  To read, call {@link #createReader()}.
75   * <p>StoreFiles may also reference store files in another Store.
76   *
77   * The reason for this weird pattern where you use a different instance for the
78   * writer and a reader is that we write once but read a lot more.
79   */
80  @InterfaceAudience.LimitedPrivate("Coprocessor")
81  public class StoreFile {
82    static final Log LOG = LogFactory.getLog(StoreFile.class.getName());
83  
84    // Keys for fileinfo values in HFile
85  
86    /** Max Sequence ID in FileInfo */
87    public static final byte [] MAX_SEQ_ID_KEY = Bytes.toBytes("MAX_SEQ_ID_KEY");
88  
89    /** Major compaction flag in FileInfo */
90    public static final byte[] MAJOR_COMPACTION_KEY =
91        Bytes.toBytes("MAJOR_COMPACTION_KEY");
92  
93    /** Minor compaction flag in FileInfo */
94    public static final byte[] EXCLUDE_FROM_MINOR_COMPACTION_KEY =
95        Bytes.toBytes("EXCLUDE_FROM_MINOR_COMPACTION");
96  
97    /** Bloom filter Type in FileInfo */
98    public static final byte[] BLOOM_FILTER_TYPE_KEY =
99        Bytes.toBytes("BLOOM_FILTER_TYPE");
100 
101   /** Delete Family Count in FileInfo */
102   public static final byte[] DELETE_FAMILY_COUNT =
103       Bytes.toBytes("DELETE_FAMILY_COUNT");
104 
105   /** Last Bloom filter key in FileInfo */
106   private static final byte[] LAST_BLOOM_KEY = Bytes.toBytes("LAST_BLOOM_KEY");
107 
108   /** Key for Timerange information in metadata*/
109   public static final byte[] TIMERANGE_KEY = Bytes.toBytes("TIMERANGE");
110 
111   /** Key for timestamp of earliest-put in metadata*/
112   public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
113 
114   private final StoreFileInfo fileInfo;
115   private final FileSystem fs;
116 
117   // Block cache configuration and reference.
118   private final CacheConfig cacheConf;
119 
120   // Keys for metadata stored in backing HFile.
121   // Set when we obtain a Reader.
122   private long sequenceid = -1;
123 
124   // max of the MemstoreTS in the KV's in this store
125   // Set when we obtain a Reader.
126   private long maxMemstoreTS = -1;
127 
128   public long getMaxMemstoreTS() {
129     return maxMemstoreTS;
130   }
131 
132   public void setMaxMemstoreTS(long maxMemstoreTS) {
133     this.maxMemstoreTS = maxMemstoreTS;
134   }
135 
136   // If true, this file was product of a major compaction.  Its then set
137   // whenever you get a Reader.
138   private AtomicBoolean majorCompaction = null;
139 
140   // If true, this file should not be included in minor compactions.
141   // It's set whenever you get a Reader.
142   private boolean excludeFromMinorCompaction = false;
143 
144   /** Meta key set when store file is a result of a bulk load */
145   public static final byte[] BULKLOAD_TASK_KEY =
146     Bytes.toBytes("BULKLOAD_SOURCE_TASK");
147   public static final byte[] BULKLOAD_TIME_KEY =
148     Bytes.toBytes("BULKLOAD_TIMESTAMP");
149 
150   /**
151    * Map of the metadata entries in the corresponding HFile
152    */
153   private Map<byte[], byte[]> metadataMap;
154 
155   // StoreFile.Reader
156   private volatile Reader reader;
157 
158   /**
159    * Bloom filter type specified in column family configuration. Does not
160    * necessarily correspond to the Bloom filter type present in the HFile.
161    */
162   private final BloomType cfBloomType;
163 
164   // the last modification time stamp
165   private long modificationTimeStamp = 0L;
166 
167   /**
168    * Constructor, loads a reader and it's indices, etc. May allocate a
169    * substantial amount of ram depending on the underlying files (10-20MB?).
170    *
171    * @param fs  The current file system to use.
172    * @param p  The path of the file.
173    * @param conf  The current configuration.
174    * @param cacheConf  The cache configuration and block cache reference.
175    * @param cfBloomType The bloom type to use for this store file as specified
176    *          by column family configuration. This may or may not be the same
177    *          as the Bloom filter type actually present in the HFile, because
178    *          column family configuration might change. If this is
179    *          {@link BloomType#NONE}, the existing Bloom filter is ignored.
180    * @throws IOException When opening the reader fails.
181    */
182   public StoreFile(final FileSystem fs, final Path p, final Configuration conf,
183         final CacheConfig cacheConf, final BloomType cfBloomType) throws IOException {
184     this(fs, new StoreFileInfo(conf, fs, p), conf, cacheConf, cfBloomType);
185   }
186 
187 
188   /**
189    * Constructor, loads a reader and it's indices, etc. May allocate a
190    * substantial amount of ram depending on the underlying files (10-20MB?).
191    *
192    * @param fs  The current file system to use.
193    * @param fileInfo  The store file information.
194    * @param conf  The current configuration.
195    * @param cacheConf  The cache configuration and block cache reference.
196    * @param cfBloomType The bloom type to use for this store file as specified
197    *          by column family configuration. This may or may not be the same
198    *          as the Bloom filter type actually present in the HFile, because
199    *          column family configuration might change. If this is
200    *          {@link BloomType#NONE}, the existing Bloom filter is ignored.
201    * @throws IOException When opening the reader fails.
202    */
203   public StoreFile(final FileSystem fs, final StoreFileInfo fileInfo, final Configuration conf,
204       final CacheConfig cacheConf,  final BloomType cfBloomType) throws IOException {
205     this.fs = fs;
206     this.fileInfo = fileInfo;
207     this.cacheConf = cacheConf;
208 
209     if (BloomFilterFactory.isGeneralBloomEnabled(conf)) {
210       this.cfBloomType = cfBloomType;
211     } else {
212       LOG.info("Ignoring bloom filter check for file " + this.getPath() + ": " +
213           "cfBloomType=" + cfBloomType + " (disabled in config)");
214       this.cfBloomType = BloomType.NONE;
215     }
216 
217     // cache the modification time stamp of this store file
218     this.modificationTimeStamp = fileInfo.getModificationTime();
219   }
220 
221   /**
222    * Clone
223    * @param other The StoreFile to clone from
224    */
225   public StoreFile(final StoreFile other) {
226     this.fs = other.fs;
227     this.fileInfo = other.fileInfo;
228     this.cacheConf = other.cacheConf;
229     this.cfBloomType = other.cfBloomType;
230     this.modificationTimeStamp = other.modificationTimeStamp;
231   }
232 
233   /**
234    * @return the StoreFile object associated to this StoreFile.
235    *         null if the StoreFile is not a reference.
236    */
237   public StoreFileInfo getFileInfo() {
238     return this.fileInfo;
239   }
240 
241   /**
242    * @return Path or null if this StoreFile was made with a Stream.
243    */
244   public Path getPath() {
245     return this.fileInfo.getPath();
246   }
247 
248   /**
249    * @return Returns the qualified path of this StoreFile
250    */
251   public Path getQualifiedPath() {
252     return this.fileInfo.getPath().makeQualified(fs);
253   }
254 
255   /**
256    * @return True if this is a StoreFile Reference; call
257    * after {@link #open(boolean canUseDropBehind)} else may get wrong answer.
258    */
259   public boolean isReference() {
260     return this.fileInfo.isReference();
261   }
262 
263   /**
264    * @return True if this is HFile.
265    */
266   public boolean isHFile() {
267     return this.fileInfo.isHFile(this.fileInfo.getPath());
268   }
269 
270   /**
271    * @return True if this file was made by a major compaction.
272    */
273   public boolean isMajorCompaction() {
274     if (this.majorCompaction == null) {
275       throw new NullPointerException("This has not been set yet");
276     }
277     return this.majorCompaction.get();
278   }
279 
280   /**
281    * @return True if this file should not be part of a minor compaction.
282    */
283   public boolean excludeFromMinorCompaction() {
284     return this.excludeFromMinorCompaction;
285   }
286 
287   /**
288    * @return This files maximum edit sequence id.
289    */
290   public long getMaxSequenceId() {
291     return this.sequenceid;
292   }
293 
294   public long getModificationTimeStamp() {
295     return modificationTimeStamp;
296   }
297 
298   public byte[] getMetadataValue(byte[] key) {
299     return metadataMap.get(key);
300   }
301 
302   /**
303    * Return the largest memstoreTS found across all storefiles in
304    * the given list. Store files that were created by a mapreduce
305    * bulk load are ignored, as they do not correspond to any specific
306    * put operation, and thus do not have a memstoreTS associated with them.
307    * @return 0 if no non-bulk-load files are provided or, this is Store that
308    * does not yet have any store files.
309    */
310   public static long getMaxMemstoreTSInList(Collection<StoreFile> sfs) {
311     long max = 0;
312     for (StoreFile sf : sfs) {
313       if (!sf.isBulkLoadResult()) {
314         max = Math.max(max, sf.getMaxMemstoreTS());
315       }
316     }
317     return max;
318   }
319 
320   /**
321    * Return the highest sequence ID found across all storefiles in
322    * the given list.
323    * @param sfs
324    * @return 0 if no non-bulk-load files are provided or, this is Store that
325    * does not yet have any store files.
326    */
327   public static long getMaxSequenceIdInList(Collection<StoreFile> sfs) {
328     long max = 0;
329     for (StoreFile sf : sfs) {
330       max = Math.max(max, sf.getMaxSequenceId());
331     }
332     return max;
333   }
334 
335   public CacheConfig getCacheConf() {
336     return this.cacheConf;
337   }
338 
339   /**
340    * Check if this storefile was created by bulk load.
341    * When a hfile is bulk loaded into HBase, we append
342    * '_SeqId_<id-when-loaded>' to the hfile name, unless
343    * "hbase.mapreduce.bulkload.assign.sequenceNumbers" is
344    * explicitly turned off.
345    * If "hbase.mapreduce.bulkload.assign.sequenceNumbers"
346    * is turned off, fall back to BULKLOAD_TIME_KEY.
347    * @return true if this storefile was created by bulk load.
348    */
349   boolean isBulkLoadResult() {
350     boolean bulkLoadedHFile = false;
351     String fileName = this.getPath().getName();
352     int startPos = fileName.indexOf("SeqId_");
353     if (startPos != -1) {
354       bulkLoadedHFile = true;
355     }
356     return metadataMap.containsKey(BULKLOAD_TIME_KEY) || bulkLoadedHFile;
357   }
358 
359   /**
360    * Return the timestamp at which this bulk load file was generated.
361    */
362   public long getBulkLoadTimestamp() {
363     byte[] bulkLoadTimestamp = metadataMap.get(BULKLOAD_TIME_KEY);
364     return (bulkLoadTimestamp == null) ? 0 : Bytes.toLong(bulkLoadTimestamp);
365   }
366 
367   /**
368    * @return the cached value of HDFS blocks distribution. The cached value is
369    * calculated when store file is opened.
370    */
371   public HDFSBlocksDistribution getHDFSBlockDistribution() {
372     return this.fileInfo.getHDFSBlockDistribution();
373   }
374 
375   /**
376    * Opens reader on this store file.  Called by Constructor.
377    * @return Reader for the store file.
378    * @throws IOException
379    * @see #closeReader(boolean)
380    */
381   private Reader open(boolean canUseDropBehind) throws IOException {
382     if (this.reader != null) {
383       throw new IllegalAccessError("Already open");
384     }
385 
386     // Open the StoreFile.Reader
387     this.reader = fileInfo.open(this.fs, this.cacheConf, canUseDropBehind);
388 
389     // Load up indices and fileinfo. This also loads Bloom filter type.
390     metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo());
391 
392     // Read in our metadata.
393     byte [] b = metadataMap.get(MAX_SEQ_ID_KEY);
394     if (b != null) {
395       // By convention, if halfhfile, top half has a sequence number > bottom
396       // half. Thats why we add one in below. Its done for case the two halves
397       // are ever merged back together --rare.  Without it, on open of store,
398       // since store files are distinguished by sequence id, the one half would
399       // subsume the other.
400       this.sequenceid = Bytes.toLong(b);
401       if (fileInfo.isTopReference()) {
402         this.sequenceid += 1;
403       }
404     }
405 
406     if (isBulkLoadResult()){
407       // generate the sequenceId from the fileName
408       // fileName is of the form <randomName>_SeqId_<id-when-loaded>_
409       String fileName = this.getPath().getName();
410       // Use lastIndexOf() to get the last, most recent bulk load seqId.
411       int startPos = fileName.lastIndexOf("SeqId_");
412       if (startPos != -1) {
413         this.sequenceid = Long.parseLong(fileName.substring(startPos + 6,
414             fileName.indexOf('_', startPos + 6)));
415         // Handle reference files as done above.
416         if (fileInfo.isTopReference()) {
417           this.sequenceid += 1;
418         }
419       }
420       this.reader.setBulkLoaded(true);
421     }
422     this.reader.setSequenceID(this.sequenceid);
423 
424     b = metadataMap.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY);
425     if (b != null) {
426       this.maxMemstoreTS = Bytes.toLong(b);
427     }
428 
429     b = metadataMap.get(MAJOR_COMPACTION_KEY);
430     if (b != null) {
431       boolean mc = Bytes.toBoolean(b);
432       if (this.majorCompaction == null) {
433         this.majorCompaction = new AtomicBoolean(mc);
434       } else {
435         this.majorCompaction.set(mc);
436       }
437     } else {
438       // Presume it is not major compacted if it doesn't explicity say so
439       // HFileOutputFormat explicitly sets the major compacted key.
440       this.majorCompaction = new AtomicBoolean(false);
441     }
442 
443     b = metadataMap.get(EXCLUDE_FROM_MINOR_COMPACTION_KEY);
444     this.excludeFromMinorCompaction = (b != null && Bytes.toBoolean(b));
445 
446     BloomType hfileBloomType = reader.getBloomFilterType();
447     if (cfBloomType != BloomType.NONE) {
448       reader.loadBloomfilter(BlockType.GENERAL_BLOOM_META);
449       if (hfileBloomType != cfBloomType) {
450         LOG.info("HFile Bloom filter type for "
451             + reader.getHFileReader().getName() + ": " + hfileBloomType
452             + ", but " + cfBloomType + " specified in column family "
453             + "configuration");
454       }
455     } else if (hfileBloomType != BloomType.NONE) {
456       LOG.info("Bloom filter turned off by CF config for "
457           + reader.getHFileReader().getName());
458     }
459 
460     // load delete family bloom filter
461     reader.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META);
462 
463     try {
464       byte [] timerangeBytes = metadataMap.get(TIMERANGE_KEY);
465       if (timerangeBytes != null) {
466         this.reader.timeRangeTracker = new TimeRangeTracker();
467         Writables.copyWritable(timerangeBytes, this.reader.timeRangeTracker);
468       }
469     } catch (IllegalArgumentException e) {
470       LOG.error("Error reading timestamp range data from meta -- " +
471           "proceeding without", e);
472       this.reader.timeRangeTracker = null;
473     }
474     return this.reader;
475   }
476 
477   public Reader createReader() throws IOException {
478     return createReader(false);
479   }
480 
481   /**
482    * @return Reader for StoreFile. creates if necessary
483    * @throws IOException
484    */
485   public Reader createReader(boolean canUseDropBehind) throws IOException {
486     if (this.reader == null) {
487       try {
488         this.reader = open(canUseDropBehind);
489       } catch (IOException e) {
490         try {
491           boolean evictOnClose =
492               cacheConf != null? cacheConf.shouldEvictOnClose(): true; 
493           this.closeReader(evictOnClose);
494         } catch (IOException ee) {
495         }
496         throw e;
497       }
498 
499     }
500     return this.reader;
501   }
502 
503   /**
504    * @return Current reader.  Must call createReader first else returns null.
505    * @see #createReader()
506    */
507   public Reader getReader() {
508     return this.reader;
509   }
510 
511   /**
512    * @param evictOnClose whether to evict blocks belonging to this file
513    * @throws IOException
514    */
515   public synchronized void closeReader(boolean evictOnClose)
516       throws IOException {
517     if (this.reader != null) {
518       this.reader.close(evictOnClose);
519       this.reader = null;
520     }
521   }
522 
523   /**
524    * Delete this file
525    * @throws IOException
526    */
527   public void deleteReader() throws IOException {
528     boolean evictOnClose =
529         cacheConf != null? cacheConf.shouldEvictOnClose(): true; 
530     closeReader(evictOnClose);
531     this.fs.delete(getPath(), true);
532   }
533 
534   @Override
535   public String toString() {
536     return this.fileInfo.toString();
537   }
538 
539   /**
540    * @return a length description of this StoreFile, suitable for debug output
541    */
542   public String toStringDetailed() {
543     StringBuilder sb = new StringBuilder();
544     sb.append(this.getPath().toString());
545     sb.append(", isReference=").append(isReference());
546     sb.append(", isBulkLoadResult=").append(isBulkLoadResult());
547     if (isBulkLoadResult()) {
548       sb.append(", bulkLoadTS=").append(getBulkLoadTimestamp());
549     } else {
550       sb.append(", seqid=").append(getMaxSequenceId());
551     }
552     sb.append(", majorCompaction=").append(isMajorCompaction());
553 
554     return sb.toString();
555   }
556 
557   public static class WriterBuilder {
558     private final Configuration conf;
559     private final CacheConfig cacheConf;
560     private final FileSystem fs;
561 
562     private KeyValue.KVComparator comparator = KeyValue.COMPARATOR;
563     private BloomType bloomType = BloomType.NONE;
564     private long maxKeyCount = 0;
565     private Path dir;
566     private Path filePath;
567     private InetSocketAddress[] favoredNodes;
568     private HFileContext fileContext;
569     private boolean shouldDropCacheBehind = false;
570 
571     public WriterBuilder(Configuration conf, CacheConfig cacheConf,
572         FileSystem fs) {
573       this.conf = conf;
574       this.cacheConf = cacheConf;
575       this.fs = fs;
576     }
577 
578     /**
579      * Use either this method or {@link #withFilePath}, but not both.
580      * @param dir Path to column family directory. The directory is created if
581      *          does not exist. The file is given a unique name within this
582      *          directory.
583      * @return this (for chained invocation)
584      */
585     public WriterBuilder withOutputDir(Path dir) {
586       Preconditions.checkNotNull(dir);
587       this.dir = dir;
588       return this;
589     }
590 
591     /**
592      * Use either this method or {@link #withOutputDir}, but not both.
593      * @param filePath the StoreFile path to write
594      * @return this (for chained invocation)
595      */
596     public WriterBuilder withFilePath(Path filePath) {
597       Preconditions.checkNotNull(filePath);
598       this.filePath = filePath;
599       return this;
600     }
601 
602     /**
603      * @param favoredNodes an array of favored nodes or possibly null
604      * @return this (for chained invocation)
605      */
606     public WriterBuilder withFavoredNodes(InetSocketAddress[] favoredNodes) {
607       this.favoredNodes = favoredNodes;
608       return this;
609     }
610 
611     public WriterBuilder withComparator(KeyValue.KVComparator comparator) {
612       Preconditions.checkNotNull(comparator);
613       this.comparator = comparator;
614       return this;
615     }
616 
617     public WriterBuilder withBloomType(BloomType bloomType) {
618       Preconditions.checkNotNull(bloomType);
619       this.bloomType = bloomType;
620       return this;
621     }
622 
623     /**
624      * @param maxKeyCount estimated maximum number of keys we expect to add
625      * @return this (for chained invocation)
626      */
627     public WriterBuilder withMaxKeyCount(long maxKeyCount) {
628       this.maxKeyCount = maxKeyCount;
629       return this;
630     }
631 
632     public WriterBuilder withFileContext(HFileContext fileContext) {
633       this.fileContext = fileContext;
634       return this;
635     }
636 
637     public WriterBuilder withShouldDropCacheBehind(boolean shouldDropCacheBehind) {
638       this.shouldDropCacheBehind = shouldDropCacheBehind;
639       return this;
640     }
641     /**
642      * Create a store file writer. Client is responsible for closing file when
643      * done. If metadata, add BEFORE closing using
644      * {@link Writer#appendMetadata}.
645      */
646     public Writer build() throws IOException {
647       if ((dir == null ? 0 : 1) + (filePath == null ? 0 : 1) != 1) {
648         throw new IllegalArgumentException("Either specify parent directory " +
649             "or file path");
650       }
651 
652       if (dir == null) {
653         dir = filePath.getParent();
654       }
655 
656       if (!fs.exists(dir)) {
657         fs.mkdirs(dir);
658       }
659 
660       if (filePath == null) {
661         filePath = getUniqueFile(fs, dir);
662         if (!BloomFilterFactory.isGeneralBloomEnabled(conf)) {
663           bloomType = BloomType.NONE;
664         }
665       }
666 
667       if (comparator == null) {
668         comparator = KeyValue.COMPARATOR;
669       }
670       return new Writer(fs, filePath,
671           conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext);
672     }
673   }
674 
675   /**
676    * @param fs
677    * @param dir Directory to create file in.
678    * @return random filename inside passed <code>dir</code>
679    */
680   public static Path getUniqueFile(final FileSystem fs, final Path dir)
681       throws IOException {
682     if (!fs.getFileStatus(dir).isDir()) {
683       throw new IOException("Expecting " + dir.toString() +
684         " to be a directory");
685     }
686     return new Path(dir, UUID.randomUUID().toString().replaceAll("-", ""));
687   }
688 
689   public Long getMinimumTimestamp() {
690     return (getReader().timeRangeTracker == null) ?
691       null :
692       getReader().timeRangeTracker.getMinimumTimestamp();
693   }
694 
695   public Long getMaximumTimestamp() {
696     return (getReader().timeRangeTracker == null) ?
697       null :
698       getReader().timeRangeTracker.getMaximumTimestamp();
699   }
700 
701 
702   /**
703    * Gets the approximate mid-point of this file that is optimal for use in splitting it.
704    * @param comparator Comparator used to compare KVs.
705    * @return The split point row, or null if splitting is not possible, or reader is null.
706    */
707   @SuppressWarnings("deprecation")
708   byte[] getFileSplitPoint(KVComparator comparator) throws IOException {
709     if (this.reader == null) {
710       LOG.warn("Storefile " + this + " Reader is null; cannot get split point");
711       return null;
712     }
713     // Get first, last, and mid keys.  Midkey is the key that starts block
714     // in middle of hfile.  Has column and timestamp.  Need to return just
715     // the row we want to split on as midkey.
716     byte [] midkey = this.reader.midkey();
717     if (midkey != null) {
718       KeyValue mk = KeyValue.createKeyValueFromKey(midkey, 0, midkey.length);
719       byte [] fk = this.reader.getFirstKey();
720       KeyValue firstKey = KeyValue.createKeyValueFromKey(fk, 0, fk.length);
721       byte [] lk = this.reader.getLastKey();
722       KeyValue lastKey = KeyValue.createKeyValueFromKey(lk, 0, lk.length);
723       // if the midkey is the same as the first or last keys, we cannot (ever) split this region.
724       if (comparator.compareRows(mk, firstKey) == 0 || comparator.compareRows(mk, lastKey) == 0) {
725         if (LOG.isDebugEnabled()) {
726           LOG.debug("cannot split because midkey is the same as first or last row");
727         }
728         return null;
729       }
730       return mk.getRow();
731     }
732     return null;
733   }
734 
735   /**
736    * A StoreFile writer.  Use this to read/write HBase Store Files. It is package
737    * local because it is an implementation detail of the HBase regionserver.
738    */
739   public static class Writer implements Compactor.CellSink {
740     private final BloomFilterWriter generalBloomFilterWriter;
741     private final BloomFilterWriter deleteFamilyBloomFilterWriter;
742     private final BloomType bloomType;
743     private byte[] lastBloomKey;
744     private int lastBloomKeyOffset, lastBloomKeyLen;
745     private KVComparator kvComparator;
746     private KeyValue lastKv = null;
747     private long earliestPutTs = HConstants.LATEST_TIMESTAMP;
748     private KeyValue lastDeleteFamilyKV = null;
749     private long deleteFamilyCnt = 0;
750 
751 
752     /** Checksum type */
753     protected ChecksumType checksumType;
754 
755     /** Bytes per Checksum */
756     protected int bytesPerChecksum;
757 
758     TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
759     /* isTimeRangeTrackerSet keeps track if the timeRange has already been set
760      * When flushing a memstore, we set TimeRange and use this variable to
761      * indicate that it doesn't need to be calculated again while
762      * appending KeyValues.
763      * It is not set in cases of compactions when it is recalculated using only
764      * the appended KeyValues*/
765     boolean isTimeRangeTrackerSet = false;
766 
767     protected HFile.Writer writer;
768 
769     /**
770      * Creates an HFile.Writer that also write helpful meta data.
771      * @param fs file system to write to
772      * @param path file name to create
773      * @param conf user configuration
774      * @param comparator key comparator
775      * @param bloomType bloom filter setting
776      * @param maxKeys the expected maximum number of keys to be added. Was used
777      *        for Bloom filter size in {@link HFile} format version 1.
778      * @param favoredNodes
779      * @param fileContext - The HFile context
780      * @throws IOException problem writing to FS
781      */
782     private Writer(FileSystem fs, Path path,
783         final Configuration conf,
784         CacheConfig cacheConf,
785         final KVComparator comparator, BloomType bloomType, long maxKeys,
786         InetSocketAddress[] favoredNodes, HFileContext fileContext)
787             throws IOException {
788       writer = HFile.getWriterFactory(conf, cacheConf)
789           .withPath(fs, path)
790           .withComparator(comparator)
791           .withFavoredNodes(favoredNodes)
792           .withFileContext(fileContext)
793           .create();
794 
795       this.kvComparator = comparator;
796 
797       generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
798           conf, cacheConf, bloomType,
799           (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
800 
801       if (generalBloomFilterWriter != null) {
802         this.bloomType = bloomType;
803         if (LOG.isTraceEnabled()) LOG.trace("Bloom filter type for " + path + ": " +
804           this.bloomType + ", " + generalBloomFilterWriter.getClass().getSimpleName());
805       } else {
806         // Not using Bloom filters.
807         this.bloomType = BloomType.NONE;
808       }
809 
810       // initialize delete family Bloom filter when there is NO RowCol Bloom
811       // filter
812       if (this.bloomType != BloomType.ROWCOL) {
813         this.deleteFamilyBloomFilterWriter = BloomFilterFactory
814             .createDeleteBloomAtWrite(conf, cacheConf,
815                 (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
816       } else {
817         deleteFamilyBloomFilterWriter = null;
818       }
819       if (deleteFamilyBloomFilterWriter != null) {
820         if (LOG.isTraceEnabled()) LOG.trace("Delete Family Bloom filter type for " + path + ": "
821             + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
822       }
823     }
824 
825     /**
826      * Writes meta data.
827      * Call before {@link #close()} since its written as meta data to this file.
828      * @param maxSequenceId Maximum sequence id.
829      * @param majorCompaction True if this file is product of a major compaction
830      * @throws IOException problem writing to FS
831      */
832     public void appendMetadata(final long maxSequenceId, final boolean majorCompaction)
833     throws IOException {
834       writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId));
835       writer.appendFileInfo(MAJOR_COMPACTION_KEY,
836           Bytes.toBytes(majorCompaction));
837       appendTrackedTimestampsToMetadata();
838     }
839 
840     /**
841      * Add TimestampRange and earliest put timestamp to Metadata
842      */
843     public void appendTrackedTimestampsToMetadata() throws IOException {
844       appendFileInfo(TIMERANGE_KEY,WritableUtils.toByteArray(timeRangeTracker));
845       appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs));
846     }
847 
848     /**
849      * Set TimeRangeTracker
850      * @param trt
851      */
852     public void setTimeRangeTracker(final TimeRangeTracker trt) {
853       this.timeRangeTracker = trt;
854       isTimeRangeTrackerSet = true;
855     }
856 
857     /**
858      * Record the earlest Put timestamp.
859      *
860      * If the timeRangeTracker is not set,
861      * update TimeRangeTracker to include the timestamp of this key
862      * @param kv
863      */
864     public void trackTimestamps(final KeyValue kv) {
865       if (KeyValue.Type.Put.getCode() == kv.getTypeByte()) {
866         earliestPutTs = Math.min(earliestPutTs, kv.getTimestamp());
867       }
868       if (!isTimeRangeTrackerSet) {
869         timeRangeTracker.includeTimestamp(kv);
870       }
871     }
872 
873     private void appendGeneralBloomfilter(final KeyValue kv) throws IOException {
874       if (this.generalBloomFilterWriter != null) {
875         // only add to the bloom filter on a new, unique key
876         boolean newKey = true;
877         if (this.lastKv != null) {
878           switch(bloomType) {
879           case ROW:
880             newKey = ! kvComparator.matchingRows(kv, lastKv);
881             break;
882           case ROWCOL:
883             newKey = ! kvComparator.matchingRowColumn(kv, lastKv);
884             break;
885           case NONE:
886             newKey = false;
887             break;
888           default:
889             throw new IOException("Invalid Bloom filter type: " + bloomType +
890                 " (ROW or ROWCOL expected)");
891           }
892         }
893         if (newKey) {
894           /*
895            * http://2.bp.blogspot.com/_Cib_A77V54U/StZMrzaKufI/AAAAAAAAADo/ZhK7bGoJdMQ/s400/KeyValue.png
896            * Key = RowLen + Row + FamilyLen + Column [Family + Qualifier] + TimeStamp
897            *
898            * 2 Types of Filtering:
899            *  1. Row = Row
900            *  2. RowCol = Row + Qualifier
901            */
902           byte[] bloomKey;
903           int bloomKeyOffset, bloomKeyLen;
904 
905           switch (bloomType) {
906           case ROW:
907             bloomKey = kv.getBuffer();
908             bloomKeyOffset = kv.getRowOffset();
909             bloomKeyLen = kv.getRowLength();
910             break;
911           case ROWCOL:
912             // merge(row, qualifier)
913             // TODO: could save one buffer copy in case of compound Bloom
914             // filters when this involves creating a KeyValue
915             bloomKey = generalBloomFilterWriter.createBloomKey(kv.getBuffer(),
916                 kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(),
917                 kv.getQualifierOffset(), kv.getQualifierLength());
918             bloomKeyOffset = 0;
919             bloomKeyLen = bloomKey.length;
920             break;
921           default:
922             throw new IOException("Invalid Bloom filter type: " + bloomType +
923                 " (ROW or ROWCOL expected)");
924           }
925           generalBloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen);
926           if (lastBloomKey != null
927               && generalBloomFilterWriter.getComparator().compareFlatKey(bloomKey,
928                   bloomKeyOffset, bloomKeyLen, lastBloomKey,
929                   lastBloomKeyOffset, lastBloomKeyLen) <= 0) {
930             throw new IOException("Non-increasing Bloom keys: "
931                 + Bytes.toStringBinary(bloomKey, bloomKeyOffset, bloomKeyLen)
932                 + " after "
933                 + Bytes.toStringBinary(lastBloomKey, lastBloomKeyOffset,
934                     lastBloomKeyLen));
935           }
936           lastBloomKey = bloomKey;
937           lastBloomKeyOffset = bloomKeyOffset;
938           lastBloomKeyLen = bloomKeyLen;
939           this.lastKv = kv;
940         }
941       }
942     }
943 
944     private void appendDeleteFamilyBloomFilter(final KeyValue kv)
945         throws IOException {
946       if (!kv.isDeleteFamily() && !kv.isDeleteFamilyVersion()) {
947         return;
948       }
949 
950       // increase the number of delete family in the store file
951       deleteFamilyCnt++;
952       if (null != this.deleteFamilyBloomFilterWriter) {
953         boolean newKey = true;
954         if (lastDeleteFamilyKV != null) {
955           newKey = !kvComparator.matchingRows(kv, lastDeleteFamilyKV);
956         }
957         if (newKey) {
958           this.deleteFamilyBloomFilterWriter.add(kv.getBuffer(),
959               kv.getRowOffset(), kv.getRowLength());
960           this.lastDeleteFamilyKV = kv;
961         }
962       }
963     }
964 
965     public void append(final KeyValue kv) throws IOException {
966       appendGeneralBloomfilter(kv);
967       appendDeleteFamilyBloomFilter(kv);
968       writer.append(kv);
969       trackTimestamps(kv);
970     }
971 
972     public Path getPath() {
973       return this.writer.getPath();
974     }
975 
976     boolean hasGeneralBloom() {
977       return this.generalBloomFilterWriter != null;
978     }
979 
980     /**
981      * For unit testing only.
982      *
983      * @return the Bloom filter used by this writer.
984      */
985     BloomFilterWriter getGeneralBloomWriter() {
986       return generalBloomFilterWriter;
987     }
988 
989     private boolean closeBloomFilter(BloomFilterWriter bfw) throws IOException {
990       boolean haveBloom = (bfw != null && bfw.getKeyCount() > 0);
991       if (haveBloom) {
992         bfw.compactBloom();
993       }
994       return haveBloom;
995     }
996 
997     private boolean closeGeneralBloomFilter() throws IOException {
998       boolean hasGeneralBloom = closeBloomFilter(generalBloomFilterWriter);
999 
1000       // add the general Bloom filter writer and append file info
1001       if (hasGeneralBloom) {
1002         writer.addGeneralBloomFilter(generalBloomFilterWriter);
1003         writer.appendFileInfo(BLOOM_FILTER_TYPE_KEY,
1004             Bytes.toBytes(bloomType.toString()));
1005         if (lastBloomKey != null) {
1006           writer.appendFileInfo(LAST_BLOOM_KEY, Arrays.copyOfRange(
1007               lastBloomKey, lastBloomKeyOffset, lastBloomKeyOffset
1008                   + lastBloomKeyLen));
1009         }
1010       }
1011       return hasGeneralBloom;
1012     }
1013 
1014     private boolean closeDeleteFamilyBloomFilter() throws IOException {
1015       boolean hasDeleteFamilyBloom = closeBloomFilter(deleteFamilyBloomFilterWriter);
1016 
1017       // add the delete family Bloom filter writer
1018       if (hasDeleteFamilyBloom) {
1019         writer.addDeleteFamilyBloomFilter(deleteFamilyBloomFilterWriter);
1020       }
1021 
1022       // append file info about the number of delete family kvs
1023       // even if there is no delete family Bloom.
1024       writer.appendFileInfo(DELETE_FAMILY_COUNT,
1025           Bytes.toBytes(this.deleteFamilyCnt));
1026 
1027       return hasDeleteFamilyBloom;
1028     }
1029 
1030     public void close() throws IOException {
1031       boolean hasGeneralBloom = this.closeGeneralBloomFilter();
1032       boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
1033 
1034       writer.close();
1035 
1036       // Log final Bloom filter statistics. This needs to be done after close()
1037       // because compound Bloom filters might be finalized as part of closing.
1038       if (StoreFile.LOG.isTraceEnabled()) {
1039         StoreFile.LOG.trace((hasGeneralBloom ? "" : "NO ") + "General Bloom and " +
1040           (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily" + " was added to HFile " +
1041           getPath());
1042       }
1043 
1044     }
1045 
1046     public void appendFileInfo(byte[] key, byte[] value) throws IOException {
1047       writer.appendFileInfo(key, value);
1048     }
1049 
1050     /** For use in testing, e.g. {@link org.apache.hadoop.hbase.regionserver.CreateRandomStoreFile}
1051      */
1052     HFile.Writer getHFileWriter() {
1053       return writer;
1054     }
1055   }
1056 
1057   /**
1058    * Reader for a StoreFile.
1059    */
1060   public static class Reader {
1061     static final Log LOG = LogFactory.getLog(Reader.class.getName());
1062 
1063     protected BloomFilter generalBloomFilter = null;
1064     protected BloomFilter deleteFamilyBloomFilter = null;
1065     protected BloomType bloomFilterType;
1066     private final HFile.Reader reader;
1067     protected TimeRangeTracker timeRangeTracker = null;
1068     protected long sequenceID = -1;
1069     private byte[] lastBloomKey;
1070     private long deleteFamilyCnt = -1;
1071     private boolean bulkLoadResult = false;
1072 
1073     public Reader(FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf)
1074         throws IOException {
1075       reader = HFile.createReader(fs, path, cacheConf, conf);
1076       bloomFilterType = BloomType.NONE;
1077     }
1078 
1079     public Reader(FileSystem fs, Path path, FSDataInputStreamWrapper in, long size,
1080         CacheConfig cacheConf, Configuration conf) throws IOException {
1081       reader = HFile.createReader(fs, path, in, size, cacheConf, conf);
1082       bloomFilterType = BloomType.NONE;
1083     }
1084 
1085     /**
1086      * ONLY USE DEFAULT CONSTRUCTOR FOR UNIT TESTS
1087      */
1088     Reader() {
1089       this.reader = null;
1090     }
1091 
1092     public KVComparator getComparator() {
1093       return reader.getComparator();
1094     }
1095 
1096     /**
1097      * Get a scanner to scan over this StoreFile. Do not use
1098      * this overload if using this scanner for compactions.
1099      *
1100      * @param cacheBlocks should this scanner cache blocks?
1101      * @param pread use pread (for highly concurrent small readers)
1102      * @return a scanner
1103      */
1104     public StoreFileScanner getStoreFileScanner(boolean cacheBlocks,
1105                                                boolean pread) {
1106       return getStoreFileScanner(cacheBlocks, pread, false,
1107         // 0 is passed as readpoint because this method is only used by test
1108         // where StoreFile is directly operated upon
1109         0);
1110     }
1111 
1112     /**
1113      * Get a scanner to scan over this StoreFile.
1114      * Bulk loaded files may or may not have mvcc info.
1115      * We will consistently ignore MVCC info in bulk loaded file.
1116      * They will be visible to scanners immediately following bulk load.
1117      *
1118      * @param cacheBlocks should this scanner cache blocks?
1119      * @param pread use pread (for highly concurrent small readers)
1120      * @param isCompaction is scanner being used for compaction?
1121      * @return a scanner
1122      */
1123     public StoreFileScanner getStoreFileScanner(boolean cacheBlocks,
1124                                                boolean pread,
1125                                                boolean isCompaction, long readPt) {
1126       return new StoreFileScanner(this,
1127                                  getScanner(cacheBlocks, pread, isCompaction),
1128                                  !isCompaction, reader.hasMVCCInfo() && !this.bulkLoadResult,
1129                                  readPt);
1130     }
1131 
1132     /**
1133      * Warning: Do not write further code which depends on this call. Instead
1134      * use getStoreFileScanner() which uses the StoreFileScanner class/interface
1135      * which is the preferred way to scan a store with higher level concepts.
1136      *
1137      * @param cacheBlocks should we cache the blocks?
1138      * @param pread use pread (for concurrent small readers)
1139      * @return the underlying HFileScanner
1140      */
1141     @Deprecated
1142     public HFileScanner getScanner(boolean cacheBlocks, boolean pread) {
1143       return getScanner(cacheBlocks, pread, false);
1144     }
1145 
1146     /**
1147      * Warning: Do not write further code which depends on this call. Instead
1148      * use getStoreFileScanner() which uses the StoreFileScanner class/interface
1149      * which is the preferred way to scan a store with higher level concepts.
1150      *
1151      * @param cacheBlocks
1152      *          should we cache the blocks?
1153      * @param pread
1154      *          use pread (for concurrent small readers)
1155      * @param isCompaction
1156      *          is scanner being used for compaction?
1157      * @return the underlying HFileScanner
1158      */
1159     @Deprecated
1160     public HFileScanner getScanner(boolean cacheBlocks, boolean pread,
1161         boolean isCompaction) {
1162       return reader.getScanner(cacheBlocks, pread, isCompaction);
1163     }
1164 
1165     public void close(boolean evictOnClose) throws IOException {
1166       reader.close(evictOnClose);
1167     }
1168 
1169     /**
1170      * Check if this storeFile may contain keys within the TimeRange that
1171      * have not expired (i.e. not older than oldestUnexpiredTS).
1172      * @param scan the current scan
1173      * @param oldestUnexpiredTS the oldest timestamp that is not expired, as
1174      *          determined by the column family's TTL
1175      * @return false if queried keys definitely don't exist in this StoreFile
1176      */
1177     boolean passesTimerangeFilter(Scan scan, long oldestUnexpiredTS) {
1178       if (timeRangeTracker == null) {
1179         return true;
1180       } else {
1181         return timeRangeTracker.includesTimeRange(scan.getTimeRange()) &&
1182             timeRangeTracker.getMaximumTimestamp() >= oldestUnexpiredTS;
1183       }
1184     }
1185 
1186     /**
1187      * Checks whether the given scan passes the Bloom filter (if present). Only
1188      * checks Bloom filters for single-row or single-row-column scans. Bloom
1189      * filter checking for multi-gets is implemented as part of the store
1190      * scanner system (see {@link StoreFileScanner#seekExactly}) and uses
1191      * the lower-level API {@link #passesGeneralBloomFilter(byte[], int, int, byte[],
1192      * int, int)}.
1193      *
1194      * @param scan the scan specification. Used to determine the row, and to
1195      *          check whether this is a single-row ("get") scan.
1196      * @param columns the set of columns. Only used for row-column Bloom
1197      *          filters.
1198      * @return true if the scan with the given column set passes the Bloom
1199      *         filter, or if the Bloom filter is not applicable for the scan.
1200      *         False if the Bloom filter is applicable and the scan fails it.
1201      */
1202      boolean passesBloomFilter(Scan scan,
1203         final SortedSet<byte[]> columns) {
1204       // Multi-column non-get scans will use Bloom filters through the
1205       // lower-level API function that this function calls.
1206       if (!scan.isGetScan()) {
1207         return true;
1208       }
1209 
1210       byte[] row = scan.getStartRow();
1211       switch (this.bloomFilterType) {
1212         case ROW:
1213           return passesGeneralBloomFilter(row, 0, row.length, null, 0, 0);
1214 
1215         case ROWCOL:
1216           if (columns != null && columns.size() == 1) {
1217             byte[] column = columns.first();
1218             return passesGeneralBloomFilter(row, 0, row.length, column, 0,
1219                 column.length);
1220           }
1221 
1222           // For multi-column queries the Bloom filter is checked from the
1223           // seekExact operation.
1224           return true;
1225 
1226         default:
1227           return true;
1228       }
1229     }
1230 
1231     public boolean passesDeleteFamilyBloomFilter(byte[] row, int rowOffset,
1232         int rowLen) {
1233       // Cache Bloom filter as a local variable in case it is set to null by
1234       // another thread on an IO error.
1235       BloomFilter bloomFilter = this.deleteFamilyBloomFilter;
1236 
1237       // Empty file or there is no delete family at all
1238       if (reader.getTrailer().getEntryCount() == 0 || deleteFamilyCnt == 0) {
1239         return false;
1240       }
1241 
1242       if (bloomFilter == null) {
1243         return true;
1244       }
1245 
1246       try {
1247         if (!bloomFilter.supportsAutoLoading()) {
1248           return true;
1249         }
1250         return bloomFilter.contains(row, rowOffset, rowLen, null);
1251       } catch (IllegalArgumentException e) {
1252         LOG.error("Bad Delete Family bloom filter data -- proceeding without",
1253             e);
1254         setDeleteFamilyBloomFilterFaulty();
1255       }
1256 
1257       return true;
1258     }
1259 
1260     /**
1261      * A method for checking Bloom filters. Called directly from
1262      * StoreFileScanner in case of a multi-column query.
1263      *
1264      * @param row
1265      * @param rowOffset
1266      * @param rowLen
1267      * @param col
1268      * @param colOffset
1269      * @param colLen
1270      * @return True if passes
1271      */
1272     public boolean passesGeneralBloomFilter(byte[] row, int rowOffset,
1273         int rowLen, byte[] col, int colOffset, int colLen) {
1274       // Cache Bloom filter as a local variable in case it is set to null by
1275       // another thread on an IO error.
1276       BloomFilter bloomFilter = this.generalBloomFilter;
1277       if (bloomFilter == null) {
1278         return true;
1279       }
1280 
1281       byte[] key;
1282       switch (bloomFilterType) {
1283         case ROW:
1284           if (col != null) {
1285             throw new RuntimeException("Row-only Bloom filter called with " +
1286                 "column specified");
1287           }
1288           if (rowOffset != 0 || rowLen != row.length) {
1289               throw new AssertionError("For row-only Bloom filters the row "
1290                   + "must occupy the whole array");
1291           }
1292           key = row;
1293           break;
1294 
1295         case ROWCOL:
1296           key = bloomFilter.createBloomKey(row, rowOffset, rowLen, col,
1297               colOffset, colLen);
1298 
1299           break;
1300 
1301         default:
1302           return true;
1303       }
1304 
1305       // Empty file
1306       if (reader.getTrailer().getEntryCount() == 0)
1307         return false;
1308 
1309       try {
1310         boolean shouldCheckBloom;
1311         ByteBuffer bloom;
1312         if (bloomFilter.supportsAutoLoading()) {
1313           bloom = null;
1314           shouldCheckBloom = true;
1315         } else {
1316           bloom = reader.getMetaBlock(HFile.BLOOM_FILTER_DATA_KEY,
1317               true);
1318           shouldCheckBloom = bloom != null;
1319         }
1320 
1321         if (shouldCheckBloom) {
1322           boolean exists;
1323 
1324           // Whether the primary Bloom key is greater than the last Bloom key
1325           // from the file info. For row-column Bloom filters this is not yet
1326           // a sufficient condition to return false.
1327           boolean keyIsAfterLast = lastBloomKey != null
1328               && bloomFilter.getComparator().compareFlatKey(key, lastBloomKey) > 0;
1329 
1330           if (bloomFilterType == BloomType.ROWCOL) {
1331             // Since a Row Delete is essentially a DeleteFamily applied to all
1332             // columns, a file might be skipped if using row+col Bloom filter.
1333             // In order to ensure this file is included an additional check is
1334             // required looking only for a row bloom.
1335             byte[] rowBloomKey = bloomFilter.createBloomKey(row, rowOffset, rowLen,
1336                 null, 0, 0);
1337 
1338             if (keyIsAfterLast
1339                 && bloomFilter.getComparator().compareFlatKey(rowBloomKey,
1340                     lastBloomKey) > 0) {
1341               exists = false;
1342             } else {
1343               exists =
1344                   bloomFilter.contains(key, 0, key.length, bloom) ||
1345                   bloomFilter.contains(rowBloomKey, 0, rowBloomKey.length,
1346                       bloom);
1347             }
1348           } else {
1349             exists = !keyIsAfterLast
1350                 && bloomFilter.contains(key, 0, key.length, bloom);
1351           }
1352 
1353           return exists;
1354         }
1355       } catch (IOException e) {
1356         LOG.error("Error reading bloom filter data -- proceeding without",
1357             e);
1358         setGeneralBloomFilterFaulty();
1359       } catch (IllegalArgumentException e) {
1360         LOG.error("Bad bloom filter data -- proceeding without", e);
1361         setGeneralBloomFilterFaulty();
1362       }
1363 
1364       return true;
1365     }
1366 
1367     /**
1368      * Checks whether the given scan rowkey range overlaps with the current storefile's
1369      * @param scan the scan specification. Used to determine the rowkey range.
1370      * @return true if there is overlap, false otherwise
1371      */
1372     public boolean passesKeyRangeFilter(Scan scan) {
1373       if (this.getFirstKey() == null || this.getLastKey() == null) {
1374         // the file is empty
1375         return false;
1376       }
1377       if (Bytes.equals(scan.getStartRow(), HConstants.EMPTY_START_ROW)
1378           && Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
1379         return true;
1380       }
1381       KeyValue smallestScanKeyValue = scan.isReversed() ? KeyValue
1382           .createFirstOnRow(scan.getStopRow()) : KeyValue.createFirstOnRow(scan
1383           .getStartRow());
1384       KeyValue largestScanKeyValue = scan.isReversed() ? KeyValue
1385           .createLastOnRow(scan.getStartRow()) : KeyValue.createLastOnRow(scan
1386           .getStopRow());
1387       boolean nonOverLapping = (getComparator().compareFlatKey(
1388           this.getFirstKey(), largestScanKeyValue.getKey()) > 0 && !Bytes
1389           .equals(scan.isReversed() ? scan.getStartRow() : scan.getStopRow(),
1390               HConstants.EMPTY_END_ROW))
1391           || getComparator().compareFlatKey(this.getLastKey(),
1392               smallestScanKeyValue.getKey()) < 0;
1393       return !nonOverLapping;
1394     }
1395 
1396     public Map<byte[], byte[]> loadFileInfo() throws IOException {
1397       Map<byte [], byte []> fi = reader.loadFileInfo();
1398 
1399       byte[] b = fi.get(BLOOM_FILTER_TYPE_KEY);
1400       if (b != null) {
1401         bloomFilterType = BloomType.valueOf(Bytes.toString(b));
1402       }
1403 
1404       lastBloomKey = fi.get(LAST_BLOOM_KEY);
1405       byte[] cnt = fi.get(DELETE_FAMILY_COUNT);
1406       if (cnt != null) {
1407         deleteFamilyCnt = Bytes.toLong(cnt);
1408       }
1409 
1410       return fi;
1411     }
1412 
1413     public void loadBloomfilter() {
1414       this.loadBloomfilter(BlockType.GENERAL_BLOOM_META);
1415       this.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META);
1416     }
1417 
1418     private void loadBloomfilter(BlockType blockType) {
1419       try {
1420         if (blockType == BlockType.GENERAL_BLOOM_META) {
1421           if (this.generalBloomFilter != null)
1422             return; // Bloom has been loaded
1423 
1424           DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
1425           if (bloomMeta != null) {
1426             // sanity check for NONE Bloom filter
1427             if (bloomFilterType == BloomType.NONE) {
1428               throw new IOException(
1429                   "valid bloom filter type not found in FileInfo");
1430             } else {
1431               generalBloomFilter = BloomFilterFactory.createFromMeta(bloomMeta,
1432                   reader);
1433               if (LOG.isTraceEnabled()) {
1434                 LOG.trace("Loaded " + bloomFilterType.toString() + " "
1435                   + generalBloomFilter.getClass().getSimpleName()
1436                   + " metadata for " + reader.getName());
1437               }
1438             }
1439           }
1440         } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) {
1441           if (this.deleteFamilyBloomFilter != null)
1442             return; // Bloom has been loaded
1443 
1444           DataInput bloomMeta = reader.getDeleteBloomFilterMetadata();
1445           if (bloomMeta != null) {
1446             deleteFamilyBloomFilter = BloomFilterFactory.createFromMeta(
1447                 bloomMeta, reader);
1448             LOG.info("Loaded Delete Family Bloom ("
1449                 + deleteFamilyBloomFilter.getClass().getSimpleName()
1450                 + ") metadata for " + reader.getName());
1451           }
1452         } else {
1453           throw new RuntimeException("Block Type: " + blockType.toString()
1454               + "is not supported for Bloom filter");
1455         }
1456       } catch (IOException e) {
1457         LOG.error("Error reading bloom filter meta for " + blockType
1458             + " -- proceeding without", e);
1459         setBloomFilterFaulty(blockType);
1460       } catch (IllegalArgumentException e) {
1461         LOG.error("Bad bloom filter meta " + blockType
1462             + " -- proceeding without", e);
1463         setBloomFilterFaulty(blockType);
1464       }
1465     }
1466 
1467     private void setBloomFilterFaulty(BlockType blockType) {
1468       if (blockType == BlockType.GENERAL_BLOOM_META) {
1469         setGeneralBloomFilterFaulty();
1470       } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) {
1471         setDeleteFamilyBloomFilterFaulty();
1472       }
1473     }
1474 
1475     /**
1476      * The number of Bloom filter entries in this store file, or an estimate
1477      * thereof, if the Bloom filter is not loaded. This always returns an upper
1478      * bound of the number of Bloom filter entries.
1479      *
1480      * @return an estimate of the number of Bloom filter entries in this file
1481      */
1482     public long getFilterEntries() {
1483       return generalBloomFilter != null ? generalBloomFilter.getKeyCount()
1484           : reader.getEntries();
1485     }
1486 
1487     public void setGeneralBloomFilterFaulty() {
1488       generalBloomFilter = null;
1489     }
1490 
1491     public void setDeleteFamilyBloomFilterFaulty() {
1492       this.deleteFamilyBloomFilter = null;
1493     }
1494 
1495     public byte[] getLastKey() {
1496       return reader.getLastKey();
1497     }
1498 
1499     public byte[] getLastRowKey() {
1500       return reader.getLastRowKey();
1501     }
1502 
1503     public byte[] midkey() throws IOException {
1504       return reader.midkey();
1505     }
1506 
1507     public long length() {
1508       return reader.length();
1509     }
1510 
1511     public long getTotalUncompressedBytes() {
1512       return reader.getTrailer().getTotalUncompressedBytes();
1513     }
1514 
1515     public long getEntries() {
1516       return reader.getEntries();
1517     }
1518 
1519     public long getDeleteFamilyCnt() {
1520       return deleteFamilyCnt;
1521     }
1522 
1523     public byte[] getFirstKey() {
1524       return reader.getFirstKey();
1525     }
1526 
1527     public long indexSize() {
1528       return reader.indexSize();
1529     }
1530 
1531     public BloomType getBloomFilterType() {
1532       return this.bloomFilterType;
1533     }
1534 
1535     public long getSequenceID() {
1536       return sequenceID;
1537     }
1538 
1539     public void setSequenceID(long sequenceID) {
1540       this.sequenceID = sequenceID;
1541     }
1542 
1543     BloomFilter getGeneralBloomFilter() {
1544       return generalBloomFilter;
1545     }
1546 
1547     long getUncompressedDataIndexSize() {
1548       return reader.getTrailer().getUncompressedDataIndexSize();
1549     }
1550 
1551     public long getTotalBloomSize() {
1552       if (generalBloomFilter == null)
1553         return 0;
1554       return generalBloomFilter.getByteSize();
1555     }
1556 
1557     public int getHFileVersion() {
1558       return reader.getTrailer().getMajorVersion();
1559     }
1560 
1561     public int getHFileMinorVersion() {
1562       return reader.getTrailer().getMinorVersion();
1563     }
1564 
1565     public HFile.Reader getHFileReader() {
1566       return reader;
1567     }
1568 
1569     void disableBloomFilterForTesting() {
1570       generalBloomFilter = null;
1571       this.deleteFamilyBloomFilter = null;
1572     }
1573 
1574     public long getMaxTimestamp() {
1575       return timeRangeTracker == null ? Long.MAX_VALUE : timeRangeTracker.getMaximumTimestamp();
1576     }
1577 
1578     public void setBulkLoaded(boolean bulkLoadResult) {
1579       this.bulkLoadResult = bulkLoadResult;
1580     }
1581 
1582     public boolean isBulkLoaded() {
1583       return this.bulkLoadResult;
1584     }
1585   }
1586 
1587   /**
1588    * Useful comparators for comparing StoreFiles.
1589    */
1590   public abstract static class Comparators {
1591     /**
1592      * Comparator that compares based on the Sequence Ids of the
1593      * the StoreFiles. Bulk loads that did not request a seq ID
1594      * are given a seq id of -1; thus, they are placed before all non-
1595      * bulk loads, and bulk loads with sequence Id. Among these files,
1596      * the size is used to determine the ordering, then bulkLoadTime.
1597      * If there are ties, the path name is used as a tie-breaker.
1598      */
1599     public static final Comparator<StoreFile> SEQ_ID =
1600       Ordering.compound(ImmutableList.of(
1601           Ordering.natural().onResultOf(new GetSeqId()),
1602           Ordering.natural().onResultOf(new GetFileSize()).reverse(),
1603           Ordering.natural().onResultOf(new GetBulkTime()),
1604           Ordering.natural().onResultOf(new GetPathName())
1605       ));
1606 
1607     private static class GetSeqId implements Function<StoreFile, Long> {
1608       @Override
1609       public Long apply(StoreFile sf) {
1610         return sf.getMaxSequenceId();
1611       }
1612     }
1613 
1614     private static class GetFileSize implements Function<StoreFile, Long> {
1615       @Override
1616       public Long apply(StoreFile sf) {
1617         return sf.getReader().length();
1618       }
1619     }
1620 
1621     private static class GetBulkTime implements Function<StoreFile, Long> {
1622       @Override
1623       public Long apply(StoreFile sf) {
1624         if (!sf.isBulkLoadResult()) return Long.MAX_VALUE;
1625         return sf.getBulkLoadTimestamp();
1626       }
1627     }
1628 
1629     private static class GetPathName implements Function<StoreFile, String> {
1630       @Override
1631       public String apply(StoreFile sf) {
1632         return sf.getPath().getName();
1633       }
1634     }
1635   }
1636 }