1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.lang.management.ManagementFactory;
22 import java.lang.management.MemoryUsage;
23
24 import com.google.common.annotations.VisibleForTesting;
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
32 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
33 import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34 import org.apache.hadoop.util.StringUtils;
35
36
37
38
39 @InterfaceAudience.Private
40 public class CacheConfig {
41 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42
43
44
45
46
47 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48 "hbase.rs.cacheblocksonwrite";
49
50
51
52
53
54 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55 "hfile.block.index.cacheonwrite";
56
57
58
59
60 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61 "hfile.block.bloom.cacheonwrite";
62
63
64
65
66 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
67 "hbase.block.data.cachecompressed";
68
69
70
71
72
73 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
74 "hbase.rs.evictblocksonclose";
75
76
77
78
79 public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
80 public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
81 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
82 "hbase.bucketcache.persistent.path";
83 public static final String BUCKET_CACHE_COMBINED_KEY =
84 "hbase.bucketcache.combinedcache.enabled";
85 public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY =
86 "hbase.bucketcache.percentage.in.combinedcache";
87 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
88 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
89 "hbase.bucketcache.writer.queuelength";
90
91
92
93
94 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
95
96
97
98
99 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
100 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
101 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
102 public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
103
104
105
106
107
108 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
109 "hbase.rs.prefetchblocksonopen";
110
111
112
113
114
115
116
117 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
118
119 private static final String DROP_BEHIND_CACHE_COMPACTION_KEY =
120 "hbase.hfile.drop.behind.compaction";
121 private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = false;
122
123
124
125 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
126 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
127 public static final boolean DEFAULT_IN_MEMORY = false;
128 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
129 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
130 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
131 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
132 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
133
134
135 private final BlockCache blockCache;
136
137
138
139
140
141 private boolean cacheDataOnRead;
142
143
144 private final boolean inMemory;
145
146
147 private boolean cacheDataOnWrite;
148
149
150 private final boolean cacheIndexesOnWrite;
151
152
153 private final boolean cacheBloomsOnWrite;
154
155
156 private boolean evictOnClose;
157
158
159 private final boolean cacheDataCompressed;
160
161
162 private final boolean prefetchOnOpen;
163
164
165 private final boolean dropBehindCompaction;
166
167
168
169
170
171
172
173 public CacheConfig(Configuration conf, HColumnDescriptor family) {
174 this(CacheConfig.instantiateBlockCache(conf),
175 family.isBlockCacheEnabled(),
176 family.isInMemory(),
177
178
179 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
180 DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
181 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
182 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
183 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
184 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
185 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
186 DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
187 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
188 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
189 DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen(),
190 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
191 );
192 }
193
194
195
196
197
198
199 public CacheConfig(Configuration conf) {
200 this(CacheConfig.instantiateBlockCache(conf),
201 DEFAULT_CACHE_DATA_ON_READ,
202 DEFAULT_IN_MEMORY,
203
204 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
205 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
206 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
207 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
208 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
209 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
210 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
211 );
212 }
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227 CacheConfig(final BlockCache blockCache,
228 final boolean cacheDataOnRead, final boolean inMemory,
229 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
230 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
231 final boolean cacheDataCompressed, final boolean prefetchOnOpen,
232 final boolean dropBehindCompaction) {
233 this.blockCache = blockCache;
234 this.cacheDataOnRead = cacheDataOnRead;
235 this.inMemory = inMemory;
236 this.cacheDataOnWrite = cacheDataOnWrite;
237 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
238 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
239 this.evictOnClose = evictOnClose;
240 this.cacheDataCompressed = cacheDataCompressed;
241 this.prefetchOnOpen = prefetchOnOpen;
242 this.dropBehindCompaction = dropBehindCompaction;
243 }
244
245
246
247
248
249 public CacheConfig(CacheConfig cacheConf) {
250 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
251 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
252 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
253 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen,
254 cacheConf.dropBehindCompaction);
255 }
256
257
258
259
260 public boolean isBlockCacheEnabled() {
261 return this.blockCache != null;
262 }
263
264
265
266
267
268 public BlockCache getBlockCache() {
269 return this.blockCache;
270 }
271
272
273
274
275
276 public boolean shouldCacheDataOnRead() {
277 return isBlockCacheEnabled() && cacheDataOnRead;
278 }
279
280 public boolean shouldDropBehindCompaction() {
281 return dropBehindCompaction;
282 }
283
284
285
286
287
288
289 public boolean shouldCacheBlockOnRead(BlockCategory category) {
290 return isBlockCacheEnabled()
291 && (cacheDataOnRead ||
292 category == BlockCategory.INDEX ||
293 category == BlockCategory.BLOOM ||
294 (prefetchOnOpen &&
295 (category != BlockCategory.META &&
296 category != BlockCategory.UNKNOWN)));
297 }
298
299
300
301
302 public boolean isInMemory() {
303 return isBlockCacheEnabled() && this.inMemory;
304 }
305
306
307
308
309
310 public boolean shouldCacheDataOnWrite() {
311 return isBlockCacheEnabled() && this.cacheDataOnWrite;
312 }
313
314
315
316
317
318
319 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
320 this.cacheDataOnWrite = cacheDataOnWrite;
321 }
322
323
324
325
326
327 public boolean shouldCacheIndexesOnWrite() {
328 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
329 }
330
331
332
333
334
335 public boolean shouldCacheBloomsOnWrite() {
336 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
337 }
338
339
340
341
342
343 public boolean shouldEvictOnClose() {
344 return isBlockCacheEnabled() && this.evictOnClose;
345 }
346
347
348
349
350
351
352 public void setEvictOnClose(boolean evictOnClose) {
353 this.evictOnClose = evictOnClose;
354 }
355
356
357
358
359 public boolean shouldCacheDataCompressed() {
360 return isBlockCacheEnabled() && this.cacheDataCompressed;
361 }
362
363
364
365
366 public boolean shouldCacheCompressed(BlockCategory category) {
367 if (!isBlockCacheEnabled()) return false;
368 switch (category) {
369 case DATA:
370 return this.cacheDataCompressed;
371 default:
372 return false;
373 }
374 }
375
376
377
378
379 public boolean shouldPrefetchOnOpen() {
380 return isBlockCacheEnabled() && this.prefetchOnOpen;
381 }
382
383
384
385
386
387
388
389
390
391 public boolean shouldReadBlockFromCache(BlockType blockType) {
392 if (!isBlockCacheEnabled()) {
393 return false;
394 }
395 if (cacheDataOnRead) {
396 return true;
397 }
398 if (prefetchOnOpen) {
399 return true;
400 }
401 if (cacheDataOnWrite) {
402 return true;
403 }
404 if (blockType == null) {
405 return true;
406 }
407 if (blockType.getCategory() == BlockCategory.BLOOM ||
408 blockType.getCategory() == BlockCategory.INDEX) {
409 return true;
410 }
411 return false;
412 }
413
414
415
416
417
418 public boolean shouldLockOnCacheMiss(BlockType blockType) {
419 if (blockType == null) {
420 return true;
421 }
422 return shouldCacheBlockOnRead(blockType.getCategory());
423 }
424
425 @Override
426 public String toString() {
427 if (!isBlockCacheEnabled()) {
428 return "CacheConfig:disabled";
429 }
430 return "CacheConfig:enabled " +
431 "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
432 "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
433 "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
434 "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
435 "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
436 "[cacheDataCompressed=" + shouldCacheDataCompressed() + "] " +
437 "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
438 }
439
440
441
442
443
444
445
446 @VisibleForTesting
447 static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
448
449
450 private static boolean blockCacheDisabled = false;
451
452
453
454
455
456
457
458 private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
459 if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
460 if (blockCacheDisabled) return null;
461
462 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
463 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
464 if (cachePercentage == 0L) {
465 blockCacheDisabled = true;
466 return null;
467 }
468 if (cachePercentage > 1.0) {
469 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
470 " must be between 0.0 and 1.0, and not > 1.0");
471 }
472
473
474 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
475 long lruCacheSize = (long) (mu.getMax() * cachePercentage);
476 int blockSize = conf.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
477 long offHeapCacheSize =
478 (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
479 DirectMemoryUtils.getDirectMemorySize());
480 if (offHeapCacheSize <= 0) {
481 String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
482 float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
483
484 long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
485 * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
486
487 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
488 DEFAULT_BUCKET_CACHE_COMBINED);
489 BucketCache bucketCache = null;
490 if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
491 int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
492 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
493 int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
494 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
495 String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
496 float combinedPercentage = conf.getFloat(
497 BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
498 DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
499 String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
500 int[] bucketSizes = null;
501 if (configuredBucketSizes != null) {
502 bucketSizes = new int[configuredBucketSizes.length];
503 for (int i = 0; i < configuredBucketSizes.length; i++) {
504 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
505 }
506 }
507 if (combinedWithLru) {
508 lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
509 bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
510 }
511 try {
512 int ioErrorsTolerationDuration = conf.getInt(
513 "hbase.bucketcache.ioengine.errors.tolerated.duration",
514 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
515 bucketCache = new BucketCache(bucketCacheIOEngineName,
516 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
517 ioErrorsTolerationDuration);
518 } catch (IOException ioex) {
519 LOG.error("Can't instantiate bucket cache", ioex);
520 throw new RuntimeException(ioex);
521 }
522 }
523 LOG.info("Allocating LruBlockCache with maximum size " +
524 StringUtils.humanReadableInt(lruCacheSize));
525 LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
526 lruCache.setVictimCache(bucketCache);
527 if (bucketCache != null && combinedWithLru) {
528 GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
529 } else {
530 GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
531 }
532 } else {
533 LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
534 GLOBAL_BLOCK_CACHE_INSTANCE = new DoubleBlockCache(
535 lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
536 }
537 return GLOBAL_BLOCK_CACHE_INSTANCE;
538 }
539 }