1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.lang.management.ManagementFactory;
22 import java.lang.management.MemoryUsage;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.HColumnDescriptor;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
31 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
32 import org.apache.hadoop.hbase.regionserver.StoreFile;
33 import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34 import org.apache.hadoop.util.StringUtils;
35
36
37
38
39 @InterfaceAudience.Private
40 public class CacheConfig {
41 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42
43
44
45
46
47 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48 "hbase.rs.cacheblocksonwrite";
49
50
51
52
53
54 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55 "hfile.block.index.cacheonwrite";
56
57
58
59
60 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61 "hfile.block.bloom.cacheonwrite";
62
63
64
65
66
67 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
68 "hbase.rs.blockcache.cachedatacompressed";
69
70
71
72
73
74 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
75 "hbase.rs.evictblocksonclose";
76
77
78
79
80 public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
81 public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
82 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
83 "hbase.bucketcache.persistent.path";
84 public static final String BUCKET_CACHE_COMBINED_KEY =
85 "hbase.bucketcache.combinedcache.enabled";
86 public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY =
87 "hbase.bucketcache.percentage.in.combinedcache";
88 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
89 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
90 "hbase.bucketcache.writer.queuelength";
91
92
93
94 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
95 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
96 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
97 public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
98
99
100
101
102
103 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
104 "hbase.rs.prefetchblocksonopen";
105
106
107
108 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
109 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
110 public static final boolean DEFAULT_IN_MEMORY = false;
111 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
112 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
113 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
114 public static final boolean DEFAULT_COMPRESSED_CACHE = false;
115 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
116
117
118 private final BlockCache blockCache;
119
120
121
122
123
124 private boolean cacheDataOnRead;
125
126
127 private final boolean inMemory;
128
129
130 private boolean cacheDataOnWrite;
131
132
133 private final boolean cacheIndexesOnWrite;
134
135
136 private final boolean cacheBloomsOnWrite;
137
138
139 private boolean evictOnClose;
140
141
142 private final boolean cacheCompressed;
143
144
145 private final boolean prefetchOnOpen;
146
147
148
149
150
151
152
153 public CacheConfig(Configuration conf, HColumnDescriptor family) {
154 this(CacheConfig.instantiateBlockCache(conf),
155 family.isBlockCacheEnabled(),
156 family.isInMemory(),
157
158
159 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
160 DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
161 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
162 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
163 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
164 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
165 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
166 DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
167 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE),
168 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
169 DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen()
170 );
171 }
172
173
174
175
176
177
178 public CacheConfig(Configuration conf) {
179 this(CacheConfig.instantiateBlockCache(conf),
180 DEFAULT_CACHE_DATA_ON_READ,
181 DEFAULT_IN_MEMORY,
182
183 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
184 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
185 DEFAULT_CACHE_INDEXES_ON_WRITE),
186 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
187 DEFAULT_CACHE_BLOOMS_ON_WRITE),
188 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
189 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
190 DEFAULT_COMPRESSED_CACHE),
191 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN)
192 );
193 }
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208 CacheConfig(final BlockCache blockCache,
209 final boolean cacheDataOnRead, final boolean inMemory,
210 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
211 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
212 final boolean cacheCompressed, final boolean prefetchOnOpen) {
213 this.blockCache = blockCache;
214 this.cacheDataOnRead = cacheDataOnRead;
215 this.inMemory = inMemory;
216 this.cacheDataOnWrite = cacheDataOnWrite;
217 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
218 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
219 this.evictOnClose = evictOnClose;
220 this.cacheCompressed = cacheCompressed;
221 this.prefetchOnOpen = prefetchOnOpen;
222 }
223
224
225
226
227
228 public CacheConfig(CacheConfig cacheConf) {
229 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
230 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
231 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
232 cacheConf.cacheCompressed, cacheConf.prefetchOnOpen);
233 }
234
235
236
237
238 public boolean isBlockCacheEnabled() {
239 return this.blockCache != null;
240 }
241
242
243
244
245
246 public BlockCache getBlockCache() {
247 return this.blockCache;
248 }
249
250
251
252
253
254 public boolean shouldCacheDataOnRead() {
255 return isBlockCacheEnabled() && cacheDataOnRead;
256 }
257
258
259
260
261
262
263 public boolean shouldCacheBlockOnRead(BlockCategory category) {
264 boolean shouldCache = isBlockCacheEnabled()
265 && (cacheDataOnRead ||
266 category == BlockCategory.INDEX ||
267 category == BlockCategory.BLOOM ||
268 (prefetchOnOpen &&
269 (category != BlockCategory.META &&
270 category != BlockCategory.UNKNOWN)));
271 return shouldCache;
272 }
273
274
275
276
277 public boolean isInMemory() {
278 return isBlockCacheEnabled() && this.inMemory;
279 }
280
281
282
283
284
285 public boolean shouldCacheDataOnWrite() {
286 return isBlockCacheEnabled() && this.cacheDataOnWrite;
287 }
288
289
290
291
292
293
294 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
295 this.cacheDataOnWrite = cacheDataOnWrite;
296 }
297
298
299
300
301
302 public boolean shouldCacheIndexesOnWrite() {
303 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
304 }
305
306
307
308
309
310 public boolean shouldCacheBloomsOnWrite() {
311 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
312 }
313
314
315
316
317
318 public boolean shouldEvictOnClose() {
319 return isBlockCacheEnabled() && this.evictOnClose;
320 }
321
322
323
324
325
326
327 public void setEvictOnClose(boolean evictOnClose) {
328 this.evictOnClose = evictOnClose;
329 }
330
331
332
333
334 public boolean shouldCacheCompressed() {
335 return isBlockCacheEnabled() && this.cacheCompressed;
336 }
337
338
339
340
341 public boolean shouldPrefetchOnOpen() {
342 return isBlockCacheEnabled() && this.prefetchOnOpen;
343 }
344
345 @Override
346 public String toString() {
347 if (!isBlockCacheEnabled()) {
348 return "CacheConfig:disabled";
349 }
350 return "CacheConfig:enabled " +
351 "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
352 "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
353 "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
354 "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
355 "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
356 "[cacheCompressed=" + shouldCacheCompressed() + "]" +
357 "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
358 }
359
360
361
362
363
364
365
366 private static BlockCache globalBlockCache;
367
368
369 private static boolean blockCacheDisabled = false;
370
371
372
373
374
375
376
377 private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
378 if (globalBlockCache != null) return globalBlockCache;
379 if (blockCacheDisabled) return null;
380
381 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
382 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
383 if (cachePercentage == 0L) {
384 blockCacheDisabled = true;
385 return null;
386 }
387 if (cachePercentage > 1.0) {
388 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
389 " must be between 0.0 and 1.0, and not > 1.0");
390 }
391
392
393 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
394 long lruCacheSize = (long) (mu.getMax() * cachePercentage);
395 int blockSize = conf.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
396 long offHeapCacheSize =
397 (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
398 DirectMemoryUtils.getDirectMemorySize());
399 if (offHeapCacheSize <= 0) {
400 String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
401 float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
402
403 long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
404 * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
405
406 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
407 DEFAULT_BUCKET_CACHE_COMBINED);
408 BucketCache bucketCache = null;
409 if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
410 int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
411 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
412 int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
413 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
414 String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
415 float combinedPercentage = conf.getFloat(
416 BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
417 DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
418 if (combinedWithLru) {
419 lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
420 bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
421 }
422 try {
423 int ioErrorsTolerationDuration = conf.getInt(
424 "hbase.bucketcache.ioengine.errors.tolerated.duration",
425 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
426 bucketCache = new BucketCache(bucketCacheIOEngineName,
427 bucketCacheSize, blockSize, writerThreads, writerQueueLen, persistentPath,
428 ioErrorsTolerationDuration);
429 } catch (IOException ioex) {
430 LOG.error("Can't instantiate bucket cache", ioex);
431 throw new RuntimeException(ioex);
432 }
433 }
434 LOG.info("Allocating LruBlockCache with maximum size " +
435 StringUtils.humanReadableInt(lruCacheSize));
436 LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize);
437 lruCache.setVictimCache(bucketCache);
438 if (bucketCache != null && combinedWithLru) {
439 globalBlockCache = new CombinedBlockCache(lruCache, bucketCache);
440 } else {
441 globalBlockCache = lruCache;
442 }
443 } else {
444 globalBlockCache = new DoubleBlockCache(
445 lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
446 }
447 return globalBlockCache;
448 }
449 }