View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.lang.management.ManagementFactory;
22  import java.lang.management.MemoryUsage;
23  
24  import com.google.common.annotations.VisibleForTesting;
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
32  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
33  import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34  import org.apache.hadoop.util.StringUtils;
35  
36  /**
37   * Stores all of the cache objects and configuration for a single HFile.
38   */
39  @InterfaceAudience.Private
40  public class CacheConfig {
41    private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42  
43    /**
44     * Configuration key to cache data blocks on write. There are separate
45     * switches for bloom blocks and non-root index blocks.
46     */
47    public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48        "hbase.rs.cacheblocksonwrite";
49  
50    /**
51     * Configuration key to cache leaf and intermediate-level index blocks on
52     * write.
53     */
54    public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55        "hfile.block.index.cacheonwrite";
56  
57    /**
58     * Configuration key to cache compound bloom filter blocks on write.
59     */
60    public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61        "hfile.block.bloom.cacheonwrite";
62  
63    /**
64     * Configuration key to cache data blocks in compressed and/or encrypted format.
65     */
66    public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
67        "hbase.block.data.cachecompressed";
68  
69    /**
70     * Configuration key to evict all blocks of a given file from the block cache
71     * when the file is closed.
72     */
73    public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
74        "hbase.rs.evictblocksonclose";
75  
76    /**
77     * Configuration keys for Bucket cache
78     */
79    public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
80    public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
81    public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = 
82        "hbase.bucketcache.persistent.path";
83    public static final String BUCKET_CACHE_COMBINED_KEY = 
84        "hbase.bucketcache.combinedcache.enabled";
85    public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = 
86        "hbase.bucketcache.percentage.in.combinedcache";
87    public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
88    public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = 
89        "hbase.bucketcache.writer.queuelength";
90  
91    /**
92     * A comma-delimited array of values for use as bucket sizes.
93     */
94    public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
95  
96    /**
97     * Defaults for Bucket cache
98     */
99    public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
100   public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
101   public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
102   public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
103 
104  /**
105    * Configuration key to prefetch all blocks of a given file into the block cache
106    * when the file is opened.
107    */
108   public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
109       "hbase.rs.prefetchblocksonopen";
110 
111   /**
112    * The target block size used by blockcache instances. Defaults to
113    * {@link HConstants#DEFAULT_BLOCKSIZE}.
114    * TODO: this config point is completely wrong, as it's used to determine the
115    * target block size of BlockCache instances. Rename.
116    */
117   public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
118 
119   private static final String DROP_BEHIND_CACHE_COMPACTION_KEY =
120       "hbase.hfile.drop.behind.compaction";
121   private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = false;
122 
123   // Defaults
124 
125   public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
126   public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
127   public static final boolean DEFAULT_IN_MEMORY = false;
128   public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
129   public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
130   public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
131   public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
132   public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
133 
134   /** Local reference to the block cache, null if completely disabled */
135   private final BlockCache blockCache;
136 
137   /**
138    * Whether blocks should be cached on read (default is on if there is a
139    * cache but this can be turned off on a per-family or per-request basis)
140    */
141   private boolean cacheDataOnRead;
142 
143   /** Whether blocks should be flagged as in-memory when being cached */
144   private final boolean inMemory;
145 
146   /** Whether data blocks should be cached when new files are written */
147   private boolean cacheDataOnWrite;
148 
149   /** Whether index blocks should be cached when new files are written */
150   private final boolean cacheIndexesOnWrite;
151 
152   /** Whether compound bloom filter blocks should be cached on write */
153   private final boolean cacheBloomsOnWrite;
154 
155   /** Whether blocks of a file should be evicted when the file is closed */
156   private boolean evictOnClose;
157 
158   /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */
159   private final boolean cacheDataCompressed;
160 
161   /** Whether data blocks should be prefetched into the cache */
162   private final boolean prefetchOnOpen;
163 
164   /** Whether or not to drop file data from the OS blockcache behind a compaction */
165   private final boolean dropBehindCompaction;
166 
167   /**
168    * Create a cache configuration using the specified configuration object and
169    * family descriptor.
170    * @param conf hbase configuration
171    * @param family column family configuration
172    */
173   public CacheConfig(Configuration conf, HColumnDescriptor family) {
174     this(CacheConfig.instantiateBlockCache(conf),
175         family.isBlockCacheEnabled(),
176         family.isInMemory(),
177         // For the following flags we enable them regardless of per-schema settings
178         // if they are enabled in the global configuration.
179         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
180             DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
181         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
182             DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
183         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
184             DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
185         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
186             DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
187         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
188         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
189             DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen(),
190         conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
191      );
192   }
193 
194   /**
195    * Create a cache configuration using the specified configuration object and
196    * defaults for family level settings.
197    * @param conf hbase configuration
198    */
199   public CacheConfig(Configuration conf) {
200     this(CacheConfig.instantiateBlockCache(conf),
201         DEFAULT_CACHE_DATA_ON_READ,
202         DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
203                            // strictly from conf
204         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
205         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
206         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
207         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
208         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
209         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
210         conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
211      );
212   }
213 
214   /**
215    * Create a block cache configuration with the specified cache and
216    * configuration parameters.
217    * @param blockCache reference to block cache, null if completely disabled
218    * @param cacheDataOnRead whether data blocks should be cached on read
219    * @param inMemory whether blocks should be flagged as in-memory
220    * @param cacheDataOnWrite whether data blocks should be cached on write
221    * @param cacheIndexesOnWrite whether index blocks should be cached on write
222    * @param cacheBloomsOnWrite whether blooms should be cached on write
223    * @param evictOnClose whether blocks should be evicted when HFile is closed
224    * @param cacheDataCompressed whether to store blocks as compressed in the cache
225    * @param prefetchOnOpen whether to prefetch blocks upon open
226    */
227   CacheConfig(final BlockCache blockCache,
228       final boolean cacheDataOnRead, final boolean inMemory,
229       final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
230       final boolean cacheBloomsOnWrite, final boolean evictOnClose,
231       final boolean cacheDataCompressed, final boolean prefetchOnOpen,
232       final boolean dropBehindCompaction) {
233     this.blockCache = blockCache;
234     this.cacheDataOnRead = cacheDataOnRead;
235     this.inMemory = inMemory;
236     this.cacheDataOnWrite = cacheDataOnWrite;
237     this.cacheIndexesOnWrite = cacheIndexesOnWrite;
238     this.cacheBloomsOnWrite = cacheBloomsOnWrite;
239     this.evictOnClose = evictOnClose;
240     this.cacheDataCompressed = cacheDataCompressed;
241     this.prefetchOnOpen = prefetchOnOpen;
242     this.dropBehindCompaction = dropBehindCompaction;
243   }
244 
245   /**
246    * Constructs a cache configuration copied from the specified configuration.
247    * @param cacheConf
248    */
249   public CacheConfig(CacheConfig cacheConf) {
250     this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
251         cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
252         cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
253         cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen,
254         cacheConf.dropBehindCompaction);
255   }
256 
257   /**
258    * Checks whether the block cache is enabled.
259    */
260   public boolean isBlockCacheEnabled() {
261     return this.blockCache != null;
262   }
263 
264   /**
265    * Returns the block cache.
266    * @return the block cache, or null if caching is completely disabled
267    */
268   public BlockCache getBlockCache() {
269     return this.blockCache;
270   }
271 
272   /**
273    * Returns whether the blocks of this HFile should be cached on read or not.
274    * @return true if blocks should be cached on read, false if not
275    */
276   public boolean shouldCacheDataOnRead() {
277     return isBlockCacheEnabled() && cacheDataOnRead;
278   }
279 
280   public boolean shouldDropBehindCompaction() {
281     return dropBehindCompaction;
282   }
283 
284   /**
285    * Should we cache a block of a particular category? We always cache
286    * important blocks such as index blocks, as long as the block cache is
287    * available.
288    */
289   public boolean shouldCacheBlockOnRead(BlockCategory category) {
290     return isBlockCacheEnabled()
291         && (cacheDataOnRead ||
292             category == BlockCategory.INDEX ||
293             category == BlockCategory.BLOOM ||
294             (prefetchOnOpen &&
295                 (category != BlockCategory.META &&
296                  category != BlockCategory.UNKNOWN)));
297   }
298 
299   /**
300    * @return true if blocks in this file should be flagged as in-memory
301    */
302   public boolean isInMemory() {
303     return isBlockCacheEnabled() && this.inMemory;
304   }
305 
306   /**
307    * @return true if data blocks should be written to the cache when an HFile is
308    *         written, false if not
309    */
310   public boolean shouldCacheDataOnWrite() {
311     return isBlockCacheEnabled() && this.cacheDataOnWrite;
312   }
313 
314   /**
315    * Only used for testing.
316    * @param cacheDataOnWrite whether data blocks should be written to the cache
317    *                         when an HFile is written
318    */
319   public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
320     this.cacheDataOnWrite = cacheDataOnWrite;
321   }
322 
323   /**
324    * @return true if index blocks should be written to the cache when an HFile
325    *         is written, false if not
326    */
327   public boolean shouldCacheIndexesOnWrite() {
328     return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
329   }
330 
331   /**
332    * @return true if bloom blocks should be written to the cache when an HFile
333    *         is written, false if not
334    */
335   public boolean shouldCacheBloomsOnWrite() {
336     return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
337   }
338 
339   /**
340    * @return true if blocks should be evicted from the cache when an HFile
341    *         reader is closed, false if not
342    */
343   public boolean shouldEvictOnClose() {
344     return isBlockCacheEnabled() && this.evictOnClose;
345   }
346 
347   /**
348    * Only used for testing.
349    * @param evictOnClose whether blocks should be evicted from the cache when an
350    *                     HFile reader is closed
351    */
352   public void setEvictOnClose(boolean evictOnClose) {
353     this.evictOnClose = evictOnClose;
354   }
355 
356   /**
357    * @return true if data blocks should be compressed in the cache, false if not
358    */
359   public boolean shouldCacheDataCompressed() {
360     return isBlockCacheEnabled() && this.cacheDataCompressed;
361   }
362 
363   /**
364    * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise
365    */
366   public boolean shouldCacheCompressed(BlockCategory category) {
367     if (!isBlockCacheEnabled()) return false;
368     switch (category) {
369       case DATA:
370         return this.cacheDataCompressed;
371       default:
372         return false;
373     }
374   }
375 
376   /**
377    * @return true if blocks should be prefetched into the cache on open, false if not
378    */
379   public boolean shouldPrefetchOnOpen() {
380     return isBlockCacheEnabled() && this.prefetchOnOpen;
381   }
382 
383   /**
384    * Return true if we may find this type of block in block cache.
385    * <p>
386    * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we
387    * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in
388    * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled}
389    * configuration.
390    */
391   public boolean shouldReadBlockFromCache(BlockType blockType) {
392     if (!isBlockCacheEnabled()) {
393       return false;
394     }
395     if (cacheDataOnRead) {
396       return true;
397     }
398     if (prefetchOnOpen) {
399       return true;
400     }
401     if (cacheDataOnWrite) {
402       return true;
403     }
404     if (blockType == null) {
405       return true;
406     }
407     if (blockType.getCategory() == BlockCategory.BLOOM ||
408             blockType.getCategory() == BlockCategory.INDEX) {
409       return true;
410     }
411     return false;
412   }
413 
414   /**
415    * If we make sure the block could not be cached, we will not acquire the lock
416    * otherwise we will acquire lock
417    */
418   public boolean shouldLockOnCacheMiss(BlockType blockType) {
419     if (blockType == null) {
420       return true;
421     }
422     return shouldCacheBlockOnRead(blockType.getCategory());
423   }
424 
425   @Override
426   public String toString() {
427     if (!isBlockCacheEnabled()) {
428       return "CacheConfig:disabled";
429     }
430     return "CacheConfig:enabled " +
431       "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
432       "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
433       "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
434       "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
435       "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
436       "[cacheDataCompressed=" + shouldCacheDataCompressed() + "] " +
437       "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
438   }
439 
440   // Static block cache reference and methods
441 
442   /**
443    * Static reference to the block cache, or null if no caching should be used
444    * at all.
445    */
446   @VisibleForTesting
447   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
448 
449   /** Boolean whether we have disabled the block cache entirely. */
450   private static boolean blockCacheDisabled = false;
451 
452   /**
453    * Returns the block cache or <code>null</code> in case none should be used.
454    *
455    * @param conf  The current configuration.
456    * @return The block cache or <code>null</code>.
457    */
458   private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
459     if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
460     if (blockCacheDisabled) return null;
461 
462     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
463       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
464     if (cachePercentage == 0L) {
465       blockCacheDisabled = true;
466       return null;
467     }
468     if (cachePercentage > 1.0) {
469       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
470         " must be between 0.0 and 1.0, and not > 1.0");
471     }
472 
473     // Calculate the amount of heap to give the heap.
474     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
475     long lruCacheSize = (long) (mu.getMax() * cachePercentage);
476     int blockSize = conf.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
477     long offHeapCacheSize =
478       (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
479           DirectMemoryUtils.getDirectMemorySize());
480     if (offHeapCacheSize <= 0) {
481       String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
482       float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
483       // A percentage of max heap size or a absolute value with unit megabytes
484       long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
485           * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
486 
487       boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
488           DEFAULT_BUCKET_CACHE_COMBINED);
489       BucketCache bucketCache = null;
490       if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
491         int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
492             DEFAULT_BUCKET_CACHE_WRITER_THREADS);
493         int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
494             DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
495         String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
496         float combinedPercentage = conf.getFloat(
497             BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
498             DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
499         String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
500         int[] bucketSizes = null;
501         if (configuredBucketSizes != null) {
502           bucketSizes = new int[configuredBucketSizes.length];
503           for (int i = 0; i < configuredBucketSizes.length; i++) {
504             bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
505           }
506         }
507         if (combinedWithLru) {
508           lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
509           bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
510         }
511         try {
512           int ioErrorsTolerationDuration = conf.getInt(
513               "hbase.bucketcache.ioengine.errors.tolerated.duration",
514               BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
515           bucketCache = new BucketCache(bucketCacheIOEngineName,
516               bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
517               ioErrorsTolerationDuration);
518         } catch (IOException ioex) {
519           LOG.error("Can't instantiate bucket cache", ioex);
520           throw new RuntimeException(ioex);
521         }
522       }
523       LOG.info("Allocating LruBlockCache with maximum size " +
524         StringUtils.humanReadableInt(lruCacheSize));
525       LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
526       lruCache.setVictimCache(bucketCache);
527       if (bucketCache != null && combinedWithLru) {
528         GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
529       } else {
530         GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
531       }
532     } else {
533       LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
534       GLOBAL_BLOCK_CACHE_INSTANCE = new DoubleBlockCache(
535           lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
536     }
537     return GLOBAL_BLOCK_CACHE_INSTANCE;
538   }
539 }