View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.lang.management.ManagementFactory;
22  import java.lang.management.MemoryUsage;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.hbase.HColumnDescriptor;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
31  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
32  import org.apache.hadoop.hbase.regionserver.StoreFile;
33  import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34  import org.apache.hadoop.util.StringUtils;
35  
36  /**
37   * Stores all of the cache objects and configuration for a single HFile.
38   */
39  @InterfaceAudience.Private
40  public class CacheConfig {
41    private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42  
43    /**
44     * Configuration key to cache data blocks on write. There are separate
45     * switches for bloom blocks and non-root index blocks.
46     */
47    public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48        "hbase.rs.cacheblocksonwrite";
49  
50    /**
51     * Configuration key to cache leaf and intermediate-level index blocks on
52     * write.
53     */
54    public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55        "hfile.block.index.cacheonwrite";
56  
57    /**
58     * Configuration key to cache compound bloom filter blocks on write.
59     */
60    public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61        "hfile.block.bloom.cacheonwrite";
62  
63    /**
64     * TODO: Implement this (jgray)
65     * Configuration key to cache data blocks in compressed format.
66     */
67    public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
68        "hbase.rs.blockcache.cachedatacompressed";
69  
70    /**
71     * Configuration key to evict all blocks of a given file from the block cache
72     * when the file is closed.
73     */
74    public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
75        "hbase.rs.evictblocksonclose";
76  
77    /**
78     * Configuration keys for Bucket cache
79     */
80    public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
81    public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
82    public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = 
83        "hbase.bucketcache.persistent.path";
84    public static final String BUCKET_CACHE_COMBINED_KEY = 
85        "hbase.bucketcache.combinedcache.enabled";
86    public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = 
87        "hbase.bucketcache.percentage.in.combinedcache";
88    public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
89    public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = 
90        "hbase.bucketcache.writer.queuelength";
91    /**
92     * Defaults for Bucket cache
93     */
94    public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
95    public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
96    public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
97    public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
98  
99   /**
100    * Configuration key to prefetch all blocks of a given file into the block cache
101    * when the file is opened.
102    */
103   public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
104       "hbase.rs.prefetchblocksonopen";
105 
106   // Defaults
107 
108   public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
109   public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
110   public static final boolean DEFAULT_IN_MEMORY = false;
111   public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
112   public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
113   public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
114   public static final boolean DEFAULT_COMPRESSED_CACHE = false;
115   public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
116 
117   /** Local reference to the block cache, null if completely disabled */
118   private final BlockCache blockCache;
119 
120   /**
121    * Whether blocks should be cached on read (default is on if there is a
122    * cache but this can be turned off on a per-family or per-request basis)
123    */
124   private boolean cacheDataOnRead;
125 
126   /** Whether blocks should be flagged as in-memory when being cached */
127   private final boolean inMemory;
128 
129   /** Whether data blocks should be cached when new files are written */
130   private boolean cacheDataOnWrite;
131 
132   /** Whether index blocks should be cached when new files are written */
133   private final boolean cacheIndexesOnWrite;
134 
135   /** Whether compound bloom filter blocks should be cached on write */
136   private final boolean cacheBloomsOnWrite;
137 
138   /** Whether blocks of a file should be evicted when the file is closed */
139   private boolean evictOnClose;
140 
141   /** Whether data blocks should be stored in compressed form in the cache */
142   private final boolean cacheCompressed;
143 
144   /** Whether data blocks should be prefetched into the cache */
145   private final boolean prefetchOnOpen;
146 
147   /**
148    * Create a cache configuration using the specified configuration object and
149    * family descriptor.
150    * @param conf hbase configuration
151    * @param family column family configuration
152    */
153   public CacheConfig(Configuration conf, HColumnDescriptor family) {
154     this(CacheConfig.instantiateBlockCache(conf),
155         family.isBlockCacheEnabled(),
156         family.isInMemory(),
157         // For the following flags we enable them regardless of per-schema settings
158         // if they are enabled in the global configuration.
159         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
160             DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
161         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
162             DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
163         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
164             DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
165         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
166             DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
167         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE),
168         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
169             DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen()
170      );
171   }
172 
173   /**
174    * Create a cache configuration using the specified configuration object and
175    * defaults for family level settings.
176    * @param conf hbase configuration
177    */
178   public CacheConfig(Configuration conf) {
179     this(CacheConfig.instantiateBlockCache(conf),
180         DEFAULT_CACHE_DATA_ON_READ,
181         DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
182                            // strictly from conf
183         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
184         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
185             DEFAULT_CACHE_INDEXES_ON_WRITE),
186             conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
187                 DEFAULT_CACHE_BLOOMS_ON_WRITE),
188         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
189         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
190             DEFAULT_COMPRESSED_CACHE),
191         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN)
192      );
193   }
194 
195   /**
196    * Create a block cache configuration with the specified cache and
197    * configuration parameters.
198    * @param blockCache reference to block cache, null if completely disabled
199    * @param cacheDataOnRead whether data blocks should be cached on read
200    * @param inMemory whether blocks should be flagged as in-memory
201    * @param cacheDataOnWrite whether data blocks should be cached on write
202    * @param cacheIndexesOnWrite whether index blocks should be cached on write
203    * @param cacheBloomsOnWrite whether blooms should be cached on write
204    * @param evictOnClose whether blocks should be evicted when HFile is closed
205    * @param cacheCompressed whether to store blocks as compressed in the cache
206    * @param prefetchOnOpen whether to prefetch blocks upon open
207    */
208   CacheConfig(final BlockCache blockCache,
209       final boolean cacheDataOnRead, final boolean inMemory,
210       final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
211       final boolean cacheBloomsOnWrite, final boolean evictOnClose,
212       final boolean cacheCompressed, final boolean prefetchOnOpen) {
213     this.blockCache = blockCache;
214     this.cacheDataOnRead = cacheDataOnRead;
215     this.inMemory = inMemory;
216     this.cacheDataOnWrite = cacheDataOnWrite;
217     this.cacheIndexesOnWrite = cacheIndexesOnWrite;
218     this.cacheBloomsOnWrite = cacheBloomsOnWrite;
219     this.evictOnClose = evictOnClose;
220     this.cacheCompressed = cacheCompressed;
221     this.prefetchOnOpen = prefetchOnOpen;
222   }
223 
224   /**
225    * Constructs a cache configuration copied from the specified configuration.
226    * @param cacheConf
227    */
228   public CacheConfig(CacheConfig cacheConf) {
229     this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
230         cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
231         cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
232         cacheConf.cacheCompressed, cacheConf.prefetchOnOpen);
233   }
234 
235   /**
236    * Checks whether the block cache is enabled.
237    */
238   public boolean isBlockCacheEnabled() {
239     return this.blockCache != null;
240   }
241 
242   /**
243    * Returns the block cache.
244    * @return the block cache, or null if caching is completely disabled
245    */
246   public BlockCache getBlockCache() {
247     return this.blockCache;
248   }
249 
250   /**
251    * Returns whether the blocks of this HFile should be cached on read or not.
252    * @return true if blocks should be cached on read, false if not
253    */
254   public boolean shouldCacheDataOnRead() {
255     return isBlockCacheEnabled() && cacheDataOnRead;
256   }
257 
258   /**
259    * Should we cache a block of a particular category? We always cache
260    * important blocks such as index blocks, as long as the block cache is
261    * available.
262    */
263   public boolean shouldCacheBlockOnRead(BlockCategory category) {
264     boolean shouldCache = isBlockCacheEnabled()
265         && (cacheDataOnRead ||
266             category == BlockCategory.INDEX ||
267             category == BlockCategory.BLOOM ||
268             (prefetchOnOpen &&
269                 (category != BlockCategory.META &&
270                  category != BlockCategory.UNKNOWN)));
271     return shouldCache;
272   }
273 
274   /**
275    * @return true if blocks in this file should be flagged as in-memory
276    */
277   public boolean isInMemory() {
278     return isBlockCacheEnabled() && this.inMemory;
279   }
280 
281   /**
282    * @return true if data blocks should be written to the cache when an HFile is
283    *         written, false if not
284    */
285   public boolean shouldCacheDataOnWrite() {
286     return isBlockCacheEnabled() && this.cacheDataOnWrite;
287   }
288 
289   /**
290    * Only used for testing.
291    * @param cacheDataOnWrite whether data blocks should be written to the cache
292    *                         when an HFile is written
293    */
294   public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
295     this.cacheDataOnWrite = cacheDataOnWrite;
296   }
297 
298   /**
299    * @return true if index blocks should be written to the cache when an HFile
300    *         is written, false if not
301    */
302   public boolean shouldCacheIndexesOnWrite() {
303     return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
304   }
305 
306   /**
307    * @return true if bloom blocks should be written to the cache when an HFile
308    *         is written, false if not
309    */
310   public boolean shouldCacheBloomsOnWrite() {
311     return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
312   }
313 
314   /**
315    * @return true if blocks should be evicted from the cache when an HFile
316    *         reader is closed, false if not
317    */
318   public boolean shouldEvictOnClose() {
319     return isBlockCacheEnabled() && this.evictOnClose;
320   }
321 
322   /**
323    * Only used for testing.
324    * @param evictOnClose whether blocks should be evicted from the cache when an
325    *                     HFile reader is closed
326    */
327   public void setEvictOnClose(boolean evictOnClose) {
328     this.evictOnClose = evictOnClose;
329   }
330 
331   /**
332    * @return true if blocks should be compressed in the cache, false if not
333    */
334   public boolean shouldCacheCompressed() {
335     return isBlockCacheEnabled() && this.cacheCompressed;
336   }
337 
338   /**
339    * @return true if blocks should be prefetched into the cache on open, false if not
340    */
341   public boolean shouldPrefetchOnOpen() {
342     return isBlockCacheEnabled() && this.prefetchOnOpen;
343   }
344 
345   @Override
346   public String toString() {
347     if (!isBlockCacheEnabled()) {
348       return "CacheConfig:disabled";
349     }
350     return "CacheConfig:enabled " +
351       "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
352       "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
353       "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
354       "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
355       "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
356       "[cacheCompressed=" + shouldCacheCompressed() + "]" +
357       "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
358   }
359 
360   // Static block cache reference and methods
361 
362   /**
363    * Static reference to the block cache, or null if no caching should be used
364    * at all.
365    */
366   private static BlockCache globalBlockCache;
367 
368   /** Boolean whether we have disabled the block cache entirely. */
369   private static boolean blockCacheDisabled = false;
370 
371   /**
372    * Returns the block cache or <code>null</code> in case none should be used.
373    *
374    * @param conf  The current configuration.
375    * @return The block cache or <code>null</code>.
376    */
377   private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
378     if (globalBlockCache != null) return globalBlockCache;
379     if (blockCacheDisabled) return null;
380 
381     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
382       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
383     if (cachePercentage == 0L) {
384       blockCacheDisabled = true;
385       return null;
386     }
387     if (cachePercentage > 1.0) {
388       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
389         " must be between 0.0 and 1.0, and not > 1.0");
390     }
391 
392     // Calculate the amount of heap to give the heap.
393     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
394     long lruCacheSize = (long) (mu.getMax() * cachePercentage);
395     int blockSize = conf.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
396     long offHeapCacheSize =
397       (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
398           DirectMemoryUtils.getDirectMemorySize());
399     if (offHeapCacheSize <= 0) {
400       String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
401       float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
402       // A percentage of max heap size or a absolute value with unit megabytes
403       long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
404           * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
405 
406       boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
407           DEFAULT_BUCKET_CACHE_COMBINED);
408       BucketCache bucketCache = null;
409       if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
410         int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
411             DEFAULT_BUCKET_CACHE_WRITER_THREADS);
412         int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
413             DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
414         String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
415         float combinedPercentage = conf.getFloat(
416             BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
417             DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
418         if (combinedWithLru) {
419           lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
420           bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
421         }
422         try {
423           int ioErrorsTolerationDuration = conf.getInt(
424               "hbase.bucketcache.ioengine.errors.tolerated.duration",
425               BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
426           bucketCache = new BucketCache(bucketCacheIOEngineName,
427               bucketCacheSize, blockSize, writerThreads, writerQueueLen, persistentPath,
428               ioErrorsTolerationDuration);
429         } catch (IOException ioex) {
430           LOG.error("Can't instantiate bucket cache", ioex);
431           throw new RuntimeException(ioex);
432         }
433       }
434       LOG.info("Allocating LruBlockCache with maximum size " +
435         StringUtils.humanReadableInt(lruCacheSize));
436       LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize);
437       lruCache.setVictimCache(bucketCache);
438       if (bucketCache != null && combinedWithLru) {
439         globalBlockCache = new CombinedBlockCache(lruCache, bucketCache);
440       } else {
441         globalBlockCache = lruCache;
442       }
443     } else {
444       globalBlockCache = new DoubleBlockCache(
445           lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
446     }
447     return globalBlockCache;
448   }
449 }