View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.lang.management.ManagementFactory;
22  import java.lang.management.MemoryUsage;
23  
24  import com.google.common.annotations.VisibleForTesting;
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
32  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
33  import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34  import org.apache.hadoop.util.StringUtils;
35  
36  /**
37   * Stores all of the cache objects and configuration for a single HFile.
38   */
39  @InterfaceAudience.Private
40  public class CacheConfig {
41    private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42  
43    /**
44     * Configuration key to cache data blocks on write. There are separate
45     * switches for bloom blocks and non-root index blocks.
46     */
47    public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48        "hbase.rs.cacheblocksonwrite";
49  
50    /**
51     * Configuration key to cache leaf and intermediate-level index blocks on
52     * write.
53     */
54    public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55        "hfile.block.index.cacheonwrite";
56  
57    /**
58     * Configuration key to cache compound bloom filter blocks on write.
59     */
60    public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61        "hfile.block.bloom.cacheonwrite";
62  
63    /**
64     * Configuration key to cache data blocks in compressed and/or encrypted format.
65     */
66    public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
67        "hbase.block.data.cachecompressed";
68  
69    /**
70     * Configuration key to evict all blocks of a given file from the block cache
71     * when the file is closed.
72     */
73    public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
74        "hbase.rs.evictblocksonclose";
75  
76    /**
77     * Configuration keys for Bucket cache
78     */
79    public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
80    public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
81    public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = 
82        "hbase.bucketcache.persistent.path";
83    public static final String BUCKET_CACHE_COMBINED_KEY = 
84        "hbase.bucketcache.combinedcache.enabled";
85    public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = 
86        "hbase.bucketcache.percentage.in.combinedcache";
87    public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
88    public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = 
89        "hbase.bucketcache.writer.queuelength";
90  
91    /**
92     * A comma-delimited array of values for use as bucket sizes.
93     */
94    public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
95  
96    /**
97     * Defaults for Bucket cache
98     */
99    public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
100   public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
101   public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
102   public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
103 
104  /**
105    * Configuration key to prefetch all blocks of a given file into the block cache
106    * when the file is opened.
107    */
108   public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
109       "hbase.rs.prefetchblocksonopen";
110 
111   /**
112    * The target block size used by blockcache instances. Defaults to
113    * {@link HConstants#DEFAULT_BLOCKSIZE}.
114    * TODO: this config point is completely wrong, as it's used to determine the
115    * target block size of BlockCache instances. Rename.
116    */
117   public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
118 
119   // Defaults
120 
121   public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
122   public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
123   public static final boolean DEFAULT_IN_MEMORY = false;
124   public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
125   public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
126   public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
127   public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
128   public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
129 
130   /** Local reference to the block cache, null if completely disabled */
131   private final BlockCache blockCache;
132 
133   /**
134    * Whether blocks should be cached on read (default is on if there is a
135    * cache but this can be turned off on a per-family or per-request basis)
136    */
137   private boolean cacheDataOnRead;
138 
139   /** Whether blocks should be flagged as in-memory when being cached */
140   private final boolean inMemory;
141 
142   /** Whether data blocks should be cached when new files are written */
143   private boolean cacheDataOnWrite;
144 
145   /** Whether index blocks should be cached when new files are written */
146   private final boolean cacheIndexesOnWrite;
147 
148   /** Whether compound bloom filter blocks should be cached on write */
149   private final boolean cacheBloomsOnWrite;
150 
151   /** Whether blocks of a file should be evicted when the file is closed */
152   private boolean evictOnClose;
153 
154   /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */
155   private final boolean cacheDataCompressed;
156 
157   /** Whether data blocks should be prefetched into the cache */
158   private final boolean prefetchOnOpen;
159 
160   /**
161    * Create a cache configuration using the specified configuration object and
162    * family descriptor.
163    * @param conf hbase configuration
164    * @param family column family configuration
165    */
166   public CacheConfig(Configuration conf, HColumnDescriptor family) {
167     this(CacheConfig.instantiateBlockCache(conf),
168         family.isBlockCacheEnabled(),
169         family.isInMemory(),
170         // For the following flags we enable them regardless of per-schema settings
171         // if they are enabled in the global configuration.
172         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
173             DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
174         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
175             DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
176         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
177             DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
178         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
179             DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
180         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
181         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
182             DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen()
183      );
184   }
185 
186   /**
187    * Create a cache configuration using the specified configuration object and
188    * defaults for family level settings.
189    * @param conf hbase configuration
190    */
191   public CacheConfig(Configuration conf) {
192     this(CacheConfig.instantiateBlockCache(conf),
193         DEFAULT_CACHE_DATA_ON_READ,
194         DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
195                            // strictly from conf
196         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
197         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
198         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
199         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
200         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
201         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN)
202     );
203   }
204 
205   /**
206    * Create a block cache configuration with the specified cache and
207    * configuration parameters.
208    * @param blockCache reference to block cache, null if completely disabled
209    * @param cacheDataOnRead whether data blocks should be cached on read
210    * @param inMemory whether blocks should be flagged as in-memory
211    * @param cacheDataOnWrite whether data blocks should be cached on write
212    * @param cacheIndexesOnWrite whether index blocks should be cached on write
213    * @param cacheBloomsOnWrite whether blooms should be cached on write
214    * @param evictOnClose whether blocks should be evicted when HFile is closed
215    * @param cacheDataCompressed whether to store blocks as compressed in the cache
216    * @param prefetchOnOpen whether to prefetch blocks upon open
217    */
218   CacheConfig(final BlockCache blockCache,
219       final boolean cacheDataOnRead, final boolean inMemory,
220       final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
221       final boolean cacheBloomsOnWrite, final boolean evictOnClose,
222       final boolean cacheDataCompressed, final boolean prefetchOnOpen) {
223     this.blockCache = blockCache;
224     this.cacheDataOnRead = cacheDataOnRead;
225     this.inMemory = inMemory;
226     this.cacheDataOnWrite = cacheDataOnWrite;
227     this.cacheIndexesOnWrite = cacheIndexesOnWrite;
228     this.cacheBloomsOnWrite = cacheBloomsOnWrite;
229     this.evictOnClose = evictOnClose;
230     this.cacheDataCompressed = cacheDataCompressed;
231     this.prefetchOnOpen = prefetchOnOpen;
232   }
233 
234   /**
235    * Constructs a cache configuration copied from the specified configuration.
236    * @param cacheConf
237    */
238   public CacheConfig(CacheConfig cacheConf) {
239     this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
240         cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
241         cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
242         cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen);
243   }
244 
245   /**
246    * Checks whether the block cache is enabled.
247    */
248   public boolean isBlockCacheEnabled() {
249     return this.blockCache != null;
250   }
251 
252   /**
253    * Returns the block cache.
254    * @return the block cache, or null if caching is completely disabled
255    */
256   public BlockCache getBlockCache() {
257     return this.blockCache;
258   }
259 
260   /**
261    * Returns whether the blocks of this HFile should be cached on read or not.
262    * @return true if blocks should be cached on read, false if not
263    */
264   public boolean shouldCacheDataOnRead() {
265     return isBlockCacheEnabled() && cacheDataOnRead;
266   }
267 
268   /**
269    * Should we cache a block of a particular category? We always cache
270    * important blocks such as index blocks, as long as the block cache is
271    * available.
272    */
273   public boolean shouldCacheBlockOnRead(BlockCategory category) {
274     return isBlockCacheEnabled()
275         && (cacheDataOnRead ||
276             category == BlockCategory.INDEX ||
277             category == BlockCategory.BLOOM ||
278             (prefetchOnOpen &&
279                 (category != BlockCategory.META &&
280                  category != BlockCategory.UNKNOWN)));
281   }
282 
283   /**
284    * @return true if blocks in this file should be flagged as in-memory
285    */
286   public boolean isInMemory() {
287     return isBlockCacheEnabled() && this.inMemory;
288   }
289 
290   /**
291    * @return true if data blocks should be written to the cache when an HFile is
292    *         written, false if not
293    */
294   public boolean shouldCacheDataOnWrite() {
295     return isBlockCacheEnabled() && this.cacheDataOnWrite;
296   }
297 
298   /**
299    * Only used for testing.
300    * @param cacheDataOnWrite whether data blocks should be written to the cache
301    *                         when an HFile is written
302    */
303   public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
304     this.cacheDataOnWrite = cacheDataOnWrite;
305   }
306 
307   /**
308    * @return true if index blocks should be written to the cache when an HFile
309    *         is written, false if not
310    */
311   public boolean shouldCacheIndexesOnWrite() {
312     return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
313   }
314 
315   /**
316    * @return true if bloom blocks should be written to the cache when an HFile
317    *         is written, false if not
318    */
319   public boolean shouldCacheBloomsOnWrite() {
320     return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
321   }
322 
323   /**
324    * @return true if blocks should be evicted from the cache when an HFile
325    *         reader is closed, false if not
326    */
327   public boolean shouldEvictOnClose() {
328     return isBlockCacheEnabled() && this.evictOnClose;
329   }
330 
331   /**
332    * Only used for testing.
333    * @param evictOnClose whether blocks should be evicted from the cache when an
334    *                     HFile reader is closed
335    */
336   public void setEvictOnClose(boolean evictOnClose) {
337     this.evictOnClose = evictOnClose;
338   }
339 
340   /**
341    * @return true if data blocks should be compressed in the cache, false if not
342    */
343   public boolean shouldCacheDataCompressed() {
344     return isBlockCacheEnabled() && this.cacheDataCompressed;
345   }
346 
347   /**
348    * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise
349    */
350   public boolean shouldCacheCompressed(BlockCategory category) {
351     if (!isBlockCacheEnabled()) return false;
352     switch (category) {
353       case DATA:
354         return this.cacheDataCompressed;
355       default:
356         return false;
357     }
358   }
359 
360   /**
361    * @return true if blocks should be prefetched into the cache on open, false if not
362    */
363   public boolean shouldPrefetchOnOpen() {
364     return isBlockCacheEnabled() && this.prefetchOnOpen;
365   }
366 
367   /**
368    * Return true if we may find this type of block in block cache.
369    * <p>
370    * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we
371    * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in
372    * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled}
373    * configuration.
374    */
375   public boolean shouldReadBlockFromCache(BlockType blockType) {
376     if (!isBlockCacheEnabled()) {
377       return false;
378     }
379     if (cacheDataOnRead) {
380       return true;
381     }
382     if (prefetchOnOpen) {
383       return true;
384     }
385     if (cacheDataOnWrite) {
386       return true;
387     }
388     if (blockType == null) {
389       return true;
390     }
391     if (blockType.getCategory() == BlockCategory.BLOOM ||
392             blockType.getCategory() == BlockCategory.INDEX) {
393       return true;
394     }
395     return false;
396   }
397 
398   /**
399    * If we make sure the block could not be cached, we will not acquire the lock
400    * otherwise we will acquire lock
401    */
402   public boolean shouldLockOnCacheMiss(BlockType blockType) {
403     if (blockType == null) {
404       return true;
405     }
406     return shouldCacheBlockOnRead(blockType.getCategory());
407   }
408 
409   @Override
410   public String toString() {
411     if (!isBlockCacheEnabled()) {
412       return "CacheConfig:disabled";
413     }
414     return "CacheConfig:enabled " +
415       "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
416       "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
417       "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
418       "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
419       "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
420       "[cacheDataCompressed=" + shouldCacheDataCompressed() + "] " +
421       "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
422   }
423 
424   // Static block cache reference and methods
425 
426   /**
427    * Static reference to the block cache, or null if no caching should be used
428    * at all.
429    */
430   @VisibleForTesting
431   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
432 
433   /** Boolean whether we have disabled the block cache entirely. */
434   private static boolean blockCacheDisabled = false;
435 
436   /**
437    * Returns the block cache or <code>null</code> in case none should be used.
438    *
439    * @param conf  The current configuration.
440    * @return The block cache or <code>null</code>.
441    */
442   private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
443     if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
444     if (blockCacheDisabled) return null;
445 
446     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
447       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
448     if (cachePercentage == 0L) {
449       blockCacheDisabled = true;
450       return null;
451     }
452     if (cachePercentage > 1.0) {
453       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
454         " must be between 0.0 and 1.0, and not > 1.0");
455     }
456 
457     // Calculate the amount of heap to give the heap.
458     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
459     long lruCacheSize = (long) (mu.getMax() * cachePercentage);
460     int blockSize = conf.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
461     long offHeapCacheSize =
462       (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
463           DirectMemoryUtils.getDirectMemorySize());
464     if (offHeapCacheSize <= 0) {
465       String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
466       float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
467       // A percentage of max heap size or a absolute value with unit megabytes
468       long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
469           * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
470 
471       boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
472           DEFAULT_BUCKET_CACHE_COMBINED);
473       BucketCache bucketCache = null;
474       if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
475         int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
476             DEFAULT_BUCKET_CACHE_WRITER_THREADS);
477         int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
478             DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
479         String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
480         float combinedPercentage = conf.getFloat(
481             BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
482             DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
483         String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
484         int[] bucketSizes = null;
485         if (configuredBucketSizes != null) {
486           bucketSizes = new int[configuredBucketSizes.length];
487           for (int i = 0; i < configuredBucketSizes.length; i++) {
488             bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
489           }
490         }
491         if (combinedWithLru) {
492           lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
493           bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
494         }
495         try {
496           int ioErrorsTolerationDuration = conf.getInt(
497               "hbase.bucketcache.ioengine.errors.tolerated.duration",
498               BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
499           bucketCache = new BucketCache(bucketCacheIOEngineName,
500               bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
501               ioErrorsTolerationDuration);
502         } catch (IOException ioex) {
503           LOG.error("Can't instantiate bucket cache", ioex);
504           throw new RuntimeException(ioex);
505         }
506       }
507       LOG.info("Allocating LruBlockCache with maximum size " +
508         StringUtils.humanReadableInt(lruCacheSize));
509       LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
510       lruCache.setVictimCache(bucketCache);
511       if (bucketCache != null && combinedWithLru) {
512         GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
513       } else {
514         GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
515       }
516     } else {
517       LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
518       GLOBAL_BLOCK_CACHE_INSTANCE = new DoubleBlockCache(
519           lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
520     }
521     return GLOBAL_BLOCK_CACHE_INSTANCE;
522   }
523 }