1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.lang.management.ManagementFactory;
22 import java.lang.management.MemoryUsage;
23
24 import com.google.common.annotations.VisibleForTesting;
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
32 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
33 import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34 import org.apache.hadoop.util.StringUtils;
35
36
37
38
39 @InterfaceAudience.Private
40 public class CacheConfig {
41 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42
43
44
45
46
47 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48 "hbase.rs.cacheblocksonwrite";
49
50
51
52
53
54 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55 "hfile.block.index.cacheonwrite";
56
57
58
59
60 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61 "hfile.block.bloom.cacheonwrite";
62
63
64
65
66 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
67 "hbase.block.data.cachecompressed";
68
69
70
71
72
73 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
74 "hbase.rs.evictblocksonclose";
75
76
77
78
79 public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
80 public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
81 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
82 "hbase.bucketcache.persistent.path";
83 public static final String BUCKET_CACHE_COMBINED_KEY =
84 "hbase.bucketcache.combinedcache.enabled";
85 public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY =
86 "hbase.bucketcache.percentage.in.combinedcache";
87 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
88 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
89 "hbase.bucketcache.writer.queuelength";
90
91
92
93
94 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
95
96
97
98
99 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
100 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
101 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
102 public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
103
104
105
106
107
108 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
109 "hbase.rs.prefetchblocksonopen";
110
111
112
113
114
115
116
117 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
118
119
120
121 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
122 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
123 public static final boolean DEFAULT_IN_MEMORY = false;
124 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
125 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
126 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
127 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
128 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
129
130
131 private final BlockCache blockCache;
132
133
134
135
136
137 private boolean cacheDataOnRead;
138
139
140 private final boolean inMemory;
141
142
143 private boolean cacheDataOnWrite;
144
145
146 private final boolean cacheIndexesOnWrite;
147
148
149 private final boolean cacheBloomsOnWrite;
150
151
152 private boolean evictOnClose;
153
154
155 private final boolean cacheDataCompressed;
156
157
158 private final boolean prefetchOnOpen;
159
160
161
162
163
164
165
166 public CacheConfig(Configuration conf, HColumnDescriptor family) {
167 this(CacheConfig.instantiateBlockCache(conf),
168 family.isBlockCacheEnabled(),
169 family.isInMemory(),
170
171
172 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
173 DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
174 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
175 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
176 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
177 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
178 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
179 DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
180 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
181 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
182 DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen()
183 );
184 }
185
186
187
188
189
190
191 public CacheConfig(Configuration conf) {
192 this(CacheConfig.instantiateBlockCache(conf),
193 DEFAULT_CACHE_DATA_ON_READ,
194 DEFAULT_IN_MEMORY,
195
196 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
197 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
198 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
199 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
200 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
201 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN)
202 );
203 }
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218 CacheConfig(final BlockCache blockCache,
219 final boolean cacheDataOnRead, final boolean inMemory,
220 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
221 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
222 final boolean cacheDataCompressed, final boolean prefetchOnOpen) {
223 this.blockCache = blockCache;
224 this.cacheDataOnRead = cacheDataOnRead;
225 this.inMemory = inMemory;
226 this.cacheDataOnWrite = cacheDataOnWrite;
227 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
228 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
229 this.evictOnClose = evictOnClose;
230 this.cacheDataCompressed = cacheDataCompressed;
231 this.prefetchOnOpen = prefetchOnOpen;
232 }
233
234
235
236
237
238 public CacheConfig(CacheConfig cacheConf) {
239 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
240 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
241 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
242 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen);
243 }
244
245
246
247
248 public boolean isBlockCacheEnabled() {
249 return this.blockCache != null;
250 }
251
252
253
254
255
256 public BlockCache getBlockCache() {
257 return this.blockCache;
258 }
259
260
261
262
263
264 public boolean shouldCacheDataOnRead() {
265 return isBlockCacheEnabled() && cacheDataOnRead;
266 }
267
268
269
270
271
272
273 public boolean shouldCacheBlockOnRead(BlockCategory category) {
274 return isBlockCacheEnabled()
275 && (cacheDataOnRead ||
276 category == BlockCategory.INDEX ||
277 category == BlockCategory.BLOOM ||
278 (prefetchOnOpen &&
279 (category != BlockCategory.META &&
280 category != BlockCategory.UNKNOWN)));
281 }
282
283
284
285
286 public boolean isInMemory() {
287 return isBlockCacheEnabled() && this.inMemory;
288 }
289
290
291
292
293
294 public boolean shouldCacheDataOnWrite() {
295 return isBlockCacheEnabled() && this.cacheDataOnWrite;
296 }
297
298
299
300
301
302
303 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
304 this.cacheDataOnWrite = cacheDataOnWrite;
305 }
306
307
308
309
310
311 public boolean shouldCacheIndexesOnWrite() {
312 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
313 }
314
315
316
317
318
319 public boolean shouldCacheBloomsOnWrite() {
320 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
321 }
322
323
324
325
326
327 public boolean shouldEvictOnClose() {
328 return isBlockCacheEnabled() && this.evictOnClose;
329 }
330
331
332
333
334
335
336 public void setEvictOnClose(boolean evictOnClose) {
337 this.evictOnClose = evictOnClose;
338 }
339
340
341
342
343 public boolean shouldCacheDataCompressed() {
344 return isBlockCacheEnabled() && this.cacheDataCompressed;
345 }
346
347
348
349
350 public boolean shouldCacheCompressed(BlockCategory category) {
351 if (!isBlockCacheEnabled()) return false;
352 switch (category) {
353 case DATA:
354 return this.cacheDataCompressed;
355 default:
356 return false;
357 }
358 }
359
360
361
362
363 public boolean shouldPrefetchOnOpen() {
364 return isBlockCacheEnabled() && this.prefetchOnOpen;
365 }
366
367
368
369
370
371
372
373
374
375 public boolean shouldReadBlockFromCache(BlockType blockType) {
376 if (!isBlockCacheEnabled()) {
377 return false;
378 }
379 if (cacheDataOnRead) {
380 return true;
381 }
382 if (prefetchOnOpen) {
383 return true;
384 }
385 if (cacheDataOnWrite) {
386 return true;
387 }
388 if (blockType == null) {
389 return true;
390 }
391 if (blockType.getCategory() == BlockCategory.BLOOM ||
392 blockType.getCategory() == BlockCategory.INDEX) {
393 return true;
394 }
395 return false;
396 }
397
398
399
400
401
402 public boolean shouldLockOnCacheMiss(BlockType blockType) {
403 if (blockType == null) {
404 return true;
405 }
406 return shouldCacheBlockOnRead(blockType.getCategory());
407 }
408
409 @Override
410 public String toString() {
411 if (!isBlockCacheEnabled()) {
412 return "CacheConfig:disabled";
413 }
414 return "CacheConfig:enabled " +
415 "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
416 "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
417 "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
418 "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
419 "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
420 "[cacheDataCompressed=" + shouldCacheDataCompressed() + "] " +
421 "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
422 }
423
424
425
426
427
428
429
430 @VisibleForTesting
431 static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
432
433
434 private static boolean blockCacheDisabled = false;
435
436
437
438
439
440
441
442 private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
443 if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
444 if (blockCacheDisabled) return null;
445
446 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
447 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
448 if (cachePercentage == 0L) {
449 blockCacheDisabled = true;
450 return null;
451 }
452 if (cachePercentage > 1.0) {
453 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
454 " must be between 0.0 and 1.0, and not > 1.0");
455 }
456
457
458 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
459 long lruCacheSize = (long) (mu.getMax() * cachePercentage);
460 int blockSize = conf.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
461 long offHeapCacheSize =
462 (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
463 DirectMemoryUtils.getDirectMemorySize());
464 if (offHeapCacheSize <= 0) {
465 String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
466 float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
467
468 long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
469 * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
470
471 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
472 DEFAULT_BUCKET_CACHE_COMBINED);
473 BucketCache bucketCache = null;
474 if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
475 int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
476 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
477 int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
478 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
479 String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
480 float combinedPercentage = conf.getFloat(
481 BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
482 DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
483 String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
484 int[] bucketSizes = null;
485 if (configuredBucketSizes != null) {
486 bucketSizes = new int[configuredBucketSizes.length];
487 for (int i = 0; i < configuredBucketSizes.length; i++) {
488 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
489 }
490 }
491 if (combinedWithLru) {
492 lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
493 bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
494 }
495 try {
496 int ioErrorsTolerationDuration = conf.getInt(
497 "hbase.bucketcache.ioengine.errors.tolerated.duration",
498 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
499 bucketCache = new BucketCache(bucketCacheIOEngineName,
500 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
501 ioErrorsTolerationDuration);
502 } catch (IOException ioex) {
503 LOG.error("Can't instantiate bucket cache", ioex);
504 throw new RuntimeException(ioex);
505 }
506 }
507 LOG.info("Allocating LruBlockCache with maximum size " +
508 StringUtils.humanReadableInt(lruCacheSize));
509 LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
510 lruCache.setVictimCache(bucketCache);
511 if (bucketCache != null && combinedWithLru) {
512 GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
513 } else {
514 GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
515 }
516 } else {
517 LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
518 GLOBAL_BLOCK_CACHE_INSTANCE = new DoubleBlockCache(
519 lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
520 }
521 return GLOBAL_BLOCK_CACHE_INSTANCE;
522 }
523 }