1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
21 import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
22
23 import java.io.IOException;
24 import java.lang.management.ManagementFactory;
25 import java.lang.management.MemoryUsage;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
34 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
35 import org.apache.hadoop.util.StringUtils;
36
37 import com.google.common.annotations.VisibleForTesting;
38
39
40
41
42 @InterfaceAudience.Private
43 public class CacheConfig {
44 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
45
46
47
48
49
50 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
51 "hbase.rs.cacheblocksonwrite";
52
53
54
55
56
57 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
58 "hfile.block.index.cacheonwrite";
59
60
61
62
63 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
64 "hfile.block.bloom.cacheonwrite";
65
66
67
68
69 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
70 "hbase.block.data.cachecompressed";
71
72
73
74
75
76 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
77 "hbase.rs.evictblocksonclose";
78
79
80
81
82
83
84
85
86
87 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
88 "hbase.bucketcache.persistent.path";
89
90
91
92
93
94
95 public static final String BUCKET_CACHE_COMBINED_KEY =
96 "hbase.bucketcache.combinedcache.enabled";
97
98 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
99 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
100 "hbase.bucketcache.writer.queuelength";
101
102
103
104
105 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
106
107
108
109
110 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
111 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
112 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
113
114
115
116
117
118 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
119 "hbase.rs.prefetchblocksonopen";
120
121
122
123
124
125
126
127 public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
128
129
130
131 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
132 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
133 public static final boolean DEFAULT_IN_MEMORY = false;
134 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
135 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
136 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
137 public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
138 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
139
140
141 private final BlockCache blockCache;
142
143
144
145
146
147
148
149 private boolean cacheDataOnRead;
150
151
152 private final boolean inMemory;
153
154
155 private boolean cacheDataOnWrite;
156
157
158 private final boolean cacheIndexesOnWrite;
159
160
161 private final boolean cacheBloomsOnWrite;
162
163
164 private boolean evictOnClose;
165
166
167 private final boolean cacheDataCompressed;
168
169
170 private final boolean prefetchOnOpen;
171
172
173
174
175
176
177 private boolean cacheDataInL1;
178
179
180
181
182
183
184
185 public CacheConfig(Configuration conf, HColumnDescriptor family) {
186 this(CacheConfig.instantiateBlockCache(conf),
187 family.isBlockCacheEnabled(),
188 family.isInMemory(),
189
190
191 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
192 DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(),
193 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
194 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(),
195 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
196 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(),
197 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
198 DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(),
199 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
200 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
201 DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(),
202 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
203 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1()
204 );
205 }
206
207
208
209
210
211
212 public CacheConfig(Configuration conf) {
213 this(CacheConfig.instantiateBlockCache(conf),
214 DEFAULT_CACHE_DATA_ON_READ,
215 DEFAULT_IN_MEMORY,
216
217 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
218 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
219 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
220 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
221 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
222 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN),
223 conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1,
224 HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1)
225 );
226 }
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244 CacheConfig(final BlockCache blockCache,
245 final boolean cacheDataOnRead, final boolean inMemory,
246 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
247 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
248 final boolean cacheDataCompressed, final boolean prefetchOnOpen,
249 final boolean cacheDataInL1) {
250 this.blockCache = blockCache;
251 this.cacheDataOnRead = cacheDataOnRead;
252 this.inMemory = inMemory;
253 this.cacheDataOnWrite = cacheDataOnWrite;
254 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
255 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
256 this.evictOnClose = evictOnClose;
257 this.cacheDataCompressed = cacheDataCompressed;
258 this.prefetchOnOpen = prefetchOnOpen;
259 this.cacheDataInL1 = cacheDataInL1;
260 LOG.info(this);
261 }
262
263
264
265
266
267 public CacheConfig(CacheConfig cacheConf) {
268 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
269 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
270 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
271 cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen,
272 cacheConf.cacheDataInL1);
273 }
274
275
276
277
278 public boolean isBlockCacheEnabled() {
279 return this.blockCache != null;
280 }
281
282
283
284
285
286 public BlockCache getBlockCache() {
287 return this.blockCache;
288 }
289
290
291
292
293
294
295 public boolean shouldCacheDataOnRead() {
296 return isBlockCacheEnabled() && cacheDataOnRead;
297 }
298
299
300
301
302
303
304 public boolean shouldCacheBlockOnRead(BlockCategory category) {
305 return isBlockCacheEnabled()
306 && (cacheDataOnRead ||
307 category == BlockCategory.INDEX ||
308 category == BlockCategory.BLOOM ||
309 (prefetchOnOpen &&
310 (category != BlockCategory.META &&
311 category != BlockCategory.UNKNOWN)));
312 }
313
314
315
316
317 public boolean isInMemory() {
318 return isBlockCacheEnabled() && this.inMemory;
319 }
320
321
322
323
324 public boolean isCacheDataInL1() {
325 return isBlockCacheEnabled() && this.cacheDataInL1;
326 }
327
328
329
330
331
332 public boolean shouldCacheDataOnWrite() {
333 return isBlockCacheEnabled() && this.cacheDataOnWrite;
334 }
335
336
337
338
339
340
341 @VisibleForTesting
342 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
343 this.cacheDataOnWrite = cacheDataOnWrite;
344 }
345
346
347
348
349
350
351 @VisibleForTesting
352 public void setCacheDataInL1(boolean cacheDataInL1) {
353 this.cacheDataInL1 = cacheDataInL1;
354 }
355
356
357
358
359
360 public boolean shouldCacheIndexesOnWrite() {
361 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
362 }
363
364
365
366
367
368 public boolean shouldCacheBloomsOnWrite() {
369 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
370 }
371
372
373
374
375
376 public boolean shouldEvictOnClose() {
377 return isBlockCacheEnabled() && this.evictOnClose;
378 }
379
380
381
382
383
384
385 public void setEvictOnClose(boolean evictOnClose) {
386 this.evictOnClose = evictOnClose;
387 }
388
389
390
391
392 public boolean shouldCacheDataCompressed() {
393 return isBlockCacheEnabled() && this.cacheDataCompressed;
394 }
395
396
397
398
399 public boolean shouldCacheCompressed(BlockCategory category) {
400 if (!isBlockCacheEnabled()) return false;
401 switch (category) {
402 case DATA:
403 return this.cacheDataCompressed;
404 default:
405 return false;
406 }
407 }
408
409
410
411
412 public boolean shouldPrefetchOnOpen() {
413 return isBlockCacheEnabled() && this.prefetchOnOpen;
414 }
415
416 @Override
417 public String toString() {
418 if (!isBlockCacheEnabled()) {
419 return "CacheConfig:disabled";
420 }
421 return "blockCache=" + getBlockCache() +
422 ", cacheDataOnRead=" + shouldCacheDataOnRead() +
423 ", cacheDataOnWrite=" + shouldCacheDataOnWrite() +
424 ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() +
425 ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() +
426 ", cacheEvictOnClose=" + shouldEvictOnClose() +
427 ", cacheDataCompressed=" + shouldCacheDataCompressed() +
428 ", prefetchOnOpen=" + shouldPrefetchOnOpen();
429 }
430
431
432
433
434
435
436
437
438 @VisibleForTesting
439 static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
440
441
442 @VisibleForTesting
443 static boolean blockCacheDisabled = false;
444
445 static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) {
446 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
447 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
448 if (cachePercentage <= 0.0001f) {
449 blockCacheDisabled = true;
450 return -1;
451 }
452 if (cachePercentage > 1.0) {
453 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
454 " must be between 0.0 and 1.0, and not > 1.0");
455 }
456
457
458 return (long) (mu.getMax() * cachePercentage);
459 }
460
461
462
463
464
465
466 private static LruBlockCache getL1(final Configuration c, final MemoryUsage mu) {
467 long lruCacheSize = getLruCacheSize(c, mu);
468 if (lruCacheSize < 0) return null;
469 int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
470 LOG.info("Allocating LruBlockCache size=" +
471 StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
472 return new LruBlockCache(lruCacheSize, blockSize, true, c);
473 }
474
475
476
477
478
479
480
481 private static BucketCache getL2(final Configuration c, final MemoryUsage mu) {
482
483 String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
484 if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
485
486 int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
487 float bucketCachePercentage = c.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
488 long bucketCacheSize = (long) (bucketCachePercentage < 1? mu.getMax() * bucketCachePercentage:
489 bucketCachePercentage * 1024 * 1024);
490 if (bucketCacheSize <= 0) {
491 throw new IllegalStateException("bucketCacheSize <= 0; Check " +
492 BUCKET_CACHE_SIZE_KEY + " setting and/or server java heap size");
493 }
494 int writerThreads = c.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
495 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
496 int writerQueueLen = c.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
497 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
498 String persistentPath = c.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
499 String[] configuredBucketSizes = c.getStrings(BUCKET_CACHE_BUCKETS_KEY);
500 int [] bucketSizes = null;
501 if (configuredBucketSizes != null) {
502 bucketSizes = new int[configuredBucketSizes.length];
503 for (int i = 0; i < configuredBucketSizes.length; i++) {
504 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
505 }
506 }
507 BucketCache bucketCache = null;
508 try {
509 int ioErrorsTolerationDuration = c.getInt(
510 "hbase.bucketcache.ioengine.errors.tolerated.duration",
511 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
512
513 bucketCache = new BucketCache(bucketCacheIOEngineName,
514 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
515 ioErrorsTolerationDuration);
516 } catch (IOException ioex) {
517 LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
518 }
519 return bucketCache;
520 }
521
522
523
524
525
526
527
528
529 public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
530 if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
531 if (blockCacheDisabled) return null;
532 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
533 LruBlockCache l1 = getL1(conf, mu);
534
535 if (blockCacheDisabled) return null;
536 BucketCache l2 = getL2(conf, mu);
537 if (l2 == null) {
538 GLOBAL_BLOCK_CACHE_INSTANCE = l1;
539 } else {
540 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
541 DEFAULT_BUCKET_CACHE_COMBINED);
542 if (combinedWithLru) {
543 GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
544 } else {
545
546
547
548
549 l1.setVictimCache(l2);
550 GLOBAL_BLOCK_CACHE_INSTANCE = l1;
551 }
552 }
553 return GLOBAL_BLOCK_CACHE_INSTANCE;
554 }
555 }