1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.lang.management.ManagementFactory;
22 import java.lang.management.MemoryUsage;
23
24 import com.google.common.annotations.VisibleForTesting;
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
32 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
33 import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34 import org.apache.hadoop.util.StringUtils;
35
36
37
38
39 @InterfaceAudience.Private
40 public class CacheConfig {
41 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42
43
44
45
46
47 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48 "hbase.rs.cacheblocksonwrite";
49
50
51
52
53
54 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55 "hfile.block.index.cacheonwrite";
56
57
58
59
60 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61 "hfile.block.bloom.cacheonwrite";
62
63
64
65
66 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
67 "hbase.block.data.cachecompressed";
68
69
70
71
72
73 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
74 "hbase.rs.evictblocksonclose";
75
76
77
78
79 public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
80 public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
81 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
82 "hbase.bucketcache.persistent.path";
83 public static final String BUCKET_CACHE_COMBINED_KEY =
84 "hbase.bucketcache.combinedcache.enabled";
85 public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY =
86 "hbase.bucketcache.percentage.in.combinedcache";
87 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
88 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
89 "hbase.bucketcache.writer.queuelength";
90
91
92
93
94 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
95
96
97
98
99 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
100 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
101 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
102 public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
103
104
105
106
107
108 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
109 "hbase.rs.prefetchblocksonopen";
110
111
112
113
114
115
116
117 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
118
119 private static final String DROP_BEHIND_CACHE_COMPACTION_KEY =
120 "hbase.hfile.drop.behind.compaction";
121 private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = false;
122
123
124
125 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
126 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
127 public static final boolean DEFAULT_IN_MEMORY = false;
128 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
129 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
130 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
131 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
132 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
133
134
135 private final BlockCache blockCache;
136
137
138
139
140
141 private boolean cacheDataOnRead;
142
143
144 private final boolean inMemory;
145
146
147 private boolean cacheDataOnWrite;
148
149
150 private final boolean cacheIndexesOnWrite;
151
152
153 private final boolean cacheBloomsOnWrite;
154
155
156 private boolean evictOnClose;
157
158
159 private final boolean cacheDataCompressed;
160
161
162 private final boolean prefetchOnOpen;
163
164
165 private final boolean dropBehindCompaction;
166
167
168
169
170
171
172
173 public CacheConfig(Configuration conf, HColumnDescriptor family) {
174 this(CacheConfig.instantiateBlockCache(conf),
175 family.isBlockCacheEnabled(),
176 family.isInMemory(),
177
178
179 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
180 DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
181 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
182 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
183 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
184 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
185 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
186 DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
187 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
188 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
189 DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen(),
190 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
191 );
192 LOG.info("Created cacheConfig for " + family.getNameAsString() + ": " + this);
193 }
194
195
196
197
198
199
200
201
202 public CacheConfig(Configuration conf) {
203 this(CacheConfig.instantiateBlockCache(conf),
204 DEFAULT_CACHE_DATA_ON_READ,
205 DEFAULT_IN_MEMORY,
206
207 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
208 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
209 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
210 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
211 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
212 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
213 conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT)
214 );
215 LOG.info("Created cacheConfig: " + this);
216 }
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231 CacheConfig(final BlockCache blockCache,
232 final boolean cacheDataOnRead, final boolean inMemory,
233 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
234 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
235 final boolean cacheDataCompressed, final boolean prefetchOnOpen,
236 final boolean dropBehindCompaction) {
237 this.blockCache = blockCache;
238 this.cacheDataOnRead = cacheDataOnRead;
239 this.inMemory = inMemory;
240 this.cacheDataOnWrite = cacheDataOnWrite;
241 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
242 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
243 this.evictOnClose = evictOnClose;
244 this.cacheDataCompressed = cacheDataCompressed;
245 this.prefetchOnOpen = prefetchOnOpen;
246 this.dropBehindCompaction = dropBehindCompaction;
247 }
248
249
250
251
252
253 public CacheConfig(CacheConfig cacheConf) {
254 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
255 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
256 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
257 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen,
258 cacheConf.dropBehindCompaction);
259 }
260
261
262
263
264 public boolean isBlockCacheEnabled() {
265 return this.blockCache != null;
266 }
267
268
269
270
271
272 public BlockCache getBlockCache() {
273 return this.blockCache;
274 }
275
276
277
278
279
280 public boolean shouldCacheDataOnRead() {
281 return isBlockCacheEnabled() && cacheDataOnRead;
282 }
283
284 public boolean shouldDropBehindCompaction() {
285 return dropBehindCompaction;
286 }
287
288
289
290
291
292
293 public boolean shouldCacheBlockOnRead(BlockCategory category) {
294 return isBlockCacheEnabled()
295 && (cacheDataOnRead ||
296 category == BlockCategory.INDEX ||
297 category == BlockCategory.BLOOM ||
298 (prefetchOnOpen &&
299 (category != BlockCategory.META &&
300 category != BlockCategory.UNKNOWN)));
301 }
302
303
304
305
306 public boolean isInMemory() {
307 return isBlockCacheEnabled() && this.inMemory;
308 }
309
310
311
312
313
314 public boolean shouldCacheDataOnWrite() {
315 return isBlockCacheEnabled() && this.cacheDataOnWrite;
316 }
317
318
319
320
321
322
323 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
324 this.cacheDataOnWrite = cacheDataOnWrite;
325 }
326
327
328
329
330
331 public boolean shouldCacheIndexesOnWrite() {
332 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
333 }
334
335
336
337
338
339 public boolean shouldCacheBloomsOnWrite() {
340 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
341 }
342
343
344
345
346
347 public boolean shouldEvictOnClose() {
348 return isBlockCacheEnabled() && this.evictOnClose;
349 }
350
351
352
353
354
355
356 public void setEvictOnClose(boolean evictOnClose) {
357 this.evictOnClose = evictOnClose;
358 }
359
360
361
362
363 public boolean shouldCacheDataCompressed() {
364 return isBlockCacheEnabled() && this.cacheDataCompressed;
365 }
366
367
368
369
370 public boolean shouldCacheCompressed(BlockCategory category) {
371 if (!isBlockCacheEnabled()) return false;
372 switch (category) {
373 case DATA:
374 return this.cacheDataCompressed;
375 default:
376 return false;
377 }
378 }
379
380
381
382
383 public boolean shouldPrefetchOnOpen() {
384 return isBlockCacheEnabled() && this.prefetchOnOpen;
385 }
386
387
388
389
390
391
392
393
394
395 public boolean shouldReadBlockFromCache(BlockType blockType) {
396 if (!isBlockCacheEnabled()) {
397 return false;
398 }
399 if (cacheDataOnRead) {
400 return true;
401 }
402 if (prefetchOnOpen) {
403 return true;
404 }
405 if (cacheDataOnWrite) {
406 return true;
407 }
408 if (blockType == null) {
409 return true;
410 }
411 if (blockType.getCategory() == BlockCategory.BLOOM ||
412 blockType.getCategory() == BlockCategory.INDEX) {
413 return true;
414 }
415 return false;
416 }
417
418
419
420
421
422 public boolean shouldLockOnCacheMiss(BlockType blockType) {
423 if (blockType == null) {
424 return true;
425 }
426 return shouldCacheBlockOnRead(blockType.getCategory());
427 }
428
429 @Override
430 public String toString() {
431 if (!isBlockCacheEnabled()) {
432 return "CacheConfig:disabled";
433 }
434 return "CacheConfig:enabled " +
435 "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
436 "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
437 "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
438 "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
439 "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
440 "[cacheDataCompressed=" + shouldCacheDataCompressed() + "] " +
441 "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
442 }
443
444
445
446
447
448
449
450 @VisibleForTesting
451 static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
452
453
454 private static boolean blockCacheDisabled = false;
455
456
457
458
459
460
461
462 private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
463 if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
464 if (blockCacheDisabled) return null;
465
466 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
467 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
468 if (cachePercentage == 0L) {
469 blockCacheDisabled = true;
470 return null;
471 }
472 if (cachePercentage > 1.0) {
473 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
474 " must be between 0.0 and 1.0, and not > 1.0");
475 }
476
477
478 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
479 long lruCacheSize = (long) (mu.getMax() * cachePercentage);
480 int blockSize = conf.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
481 long offHeapCacheSize =
482 (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
483 DirectMemoryUtils.getDirectMemorySize());
484 if (offHeapCacheSize <= 0) {
485 String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
486 float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
487
488 long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
489 * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
490
491 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
492 DEFAULT_BUCKET_CACHE_COMBINED);
493 BucketCache bucketCache = null;
494 if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
495 int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
496 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
497 int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
498 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
499 String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
500 float combinedPercentage = conf.getFloat(
501 BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
502 DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
503 String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
504 int[] bucketSizes = null;
505 if (configuredBucketSizes != null) {
506 bucketSizes = new int[configuredBucketSizes.length];
507 for (int i = 0; i < configuredBucketSizes.length; i++) {
508 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
509 }
510 }
511 if (combinedWithLru) {
512 lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
513 bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
514 }
515 try {
516 int ioErrorsTolerationDuration = conf.getInt(
517 "hbase.bucketcache.ioengine.errors.tolerated.duration",
518 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
519 bucketCache = new BucketCache(bucketCacheIOEngineName,
520 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
521 ioErrorsTolerationDuration);
522 } catch (IOException ioex) {
523 LOG.error("Can't instantiate bucket cache", ioex);
524 throw new RuntimeException(ioex);
525 }
526 }
527 LOG.info("Allocating LruBlockCache with maximum size " +
528 StringUtils.humanReadableInt(lruCacheSize));
529 LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
530 lruCache.setVictimCache(bucketCache);
531 if (bucketCache != null && combinedWithLru) {
532 GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
533 } else {
534 GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
535 }
536 } else {
537 LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
538 GLOBAL_BLOCK_CACHE_INSTANCE = new DoubleBlockCache(
539 lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
540 }
541 return GLOBAL_BLOCK_CACHE_INSTANCE;
542 }
543 }