1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.lang.management.ManagementFactory;
21 import java.lang.management.MemoryUsage;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.hbase.HColumnDescriptor;
27 import org.apache.hadoop.hbase.HConstants;
28 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
29 import org.apache.hadoop.hbase.regionserver.StoreFile;
30 import org.apache.hadoop.hbase.util.DirectMemoryUtils;
31 import org.apache.hadoop.util.StringUtils;
32
33
34
35
36 public class CacheConfig {
37 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
38
39
40
41
42
43 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
44 "hbase.rs.cacheblocksonwrite";
45
46
47
48
49
50 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
51 "hfile.block.index.cacheonwrite";
52
53
54
55
56 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
57 "hfile.block.bloom.cacheonwrite";
58
59
60
61
62
63 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
64 "hbase.rs.blockcache.cachedatacompressed";
65
66
67
68
69
70 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
71 "hbase.rs.evictblocksonclose";
72
73
74
75 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
76 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
77 public static final boolean DEFAULT_IN_MEMORY = false;
78 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
79 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
80 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
81 public static final boolean DEFAULT_COMPRESSED_CACHE = false;
82
83
84 private final BlockCache blockCache;
85
86
87
88
89
90 private boolean cacheDataOnRead;
91
92
93 private final boolean inMemory;
94
95
96 private boolean cacheDataOnWrite;
97
98
99 private final boolean cacheIndexesOnWrite;
100
101
102 private final boolean cacheBloomsOnWrite;
103
104
105 private boolean evictOnClose;
106
107
108 private final boolean cacheCompressed;
109
110
111
112
113
114
115
116 public CacheConfig(Configuration conf, HColumnDescriptor family) {
117 this(CacheConfig.instantiateBlockCache(conf),
118 family.isBlockCacheEnabled(),
119 family.isInMemory(),
120
121
122 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
123 DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
124 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
125 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
126 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
127 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
128 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
129 DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
130 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE)
131 );
132 }
133
134
135
136
137
138
139 public CacheConfig(Configuration conf) {
140 this(CacheConfig.instantiateBlockCache(conf),
141 DEFAULT_CACHE_DATA_ON_READ,
142 DEFAULT_IN_MEMORY,
143
144 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
145 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
146 DEFAULT_CACHE_INDEXES_ON_WRITE),
147 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
148 DEFAULT_CACHE_BLOOMS_ON_WRITE),
149 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
150 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
151 DEFAULT_COMPRESSED_CACHE)
152 );
153 }
154
155
156
157
158
159
160
161
162
163
164
165
166
167 CacheConfig(final BlockCache blockCache,
168 final boolean cacheDataOnRead, final boolean inMemory,
169 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
170 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
171 final boolean cacheCompressed) {
172 this.blockCache = blockCache;
173 this.cacheDataOnRead = cacheDataOnRead;
174 this.inMemory = inMemory;
175 this.cacheDataOnWrite = cacheDataOnWrite;
176 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
177 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
178 this.evictOnClose = evictOnClose;
179 this.cacheCompressed = cacheCompressed;
180 }
181
182
183
184
185
186 public CacheConfig(CacheConfig cacheConf) {
187 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
188 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
189 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
190 cacheConf.cacheCompressed);
191 }
192
193
194
195
196 public boolean isBlockCacheEnabled() {
197 return this.blockCache != null;
198 }
199
200
201
202
203
204 public BlockCache getBlockCache() {
205 return this.blockCache;
206 }
207
208
209
210
211
212 public boolean shouldCacheDataOnRead() {
213 return isBlockCacheEnabled() && cacheDataOnRead;
214 }
215
216
217
218
219
220
221 public boolean shouldCacheBlockOnRead(BlockCategory category) {
222 boolean shouldCache = isBlockCacheEnabled()
223 && (cacheDataOnRead ||
224 category == BlockCategory.INDEX ||
225 category == BlockCategory.BLOOM);
226 return shouldCache;
227 }
228
229
230
231
232 public boolean isInMemory() {
233 return isBlockCacheEnabled() && this.inMemory;
234 }
235
236
237
238
239
240 public boolean shouldCacheDataOnWrite() {
241 return isBlockCacheEnabled() && this.cacheDataOnWrite;
242 }
243
244
245
246
247
248
249 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
250 this.cacheDataOnWrite = cacheDataOnWrite;
251 }
252
253
254
255
256
257 public boolean shouldCacheIndexesOnWrite() {
258 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
259 }
260
261
262
263
264
265 public boolean shouldCacheBloomsOnWrite() {
266 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
267 }
268
269
270
271
272
273 public boolean shouldEvictOnClose() {
274 return isBlockCacheEnabled() && this.evictOnClose;
275 }
276
277
278
279
280
281
282 public void setEvictOnClose(boolean evictOnClose) {
283 this.evictOnClose = evictOnClose;
284 }
285
286
287
288
289 public boolean shouldCacheCompressed() {
290 return isBlockCacheEnabled() && this.cacheCompressed;
291 }
292
293 @Override
294 public String toString() {
295 if (!isBlockCacheEnabled()) {
296 return "CacheConfig:disabled";
297 }
298 return "CacheConfig:enabled " +
299 "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
300 "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
301 "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
302 "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
303 "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
304 "[cacheCompressed=" + shouldCacheCompressed() + "]";
305 }
306
307
308
309
310
311
312
313 private static BlockCache globalBlockCache;
314
315
316 private static boolean blockCacheDisabled = false;
317
318
319
320
321
322
323
324 private static synchronized BlockCache instantiateBlockCache(
325 Configuration conf) {
326 if (globalBlockCache != null) return globalBlockCache;
327 if (blockCacheDisabled) return null;
328
329 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
330 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
331 if (cachePercentage == 0L) {
332 blockCacheDisabled = true;
333 return null;
334 }
335 if (cachePercentage > 1.0) {
336 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
337 " must be between 0.0 and 1.0, and not > 1.0");
338 }
339
340
341 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
342 long cacheSize = (long)(mu.getMax() * cachePercentage);
343 int blockSize = conf.getInt("hbase.offheapcache.minblocksize",
344 HFile.DEFAULT_BLOCKSIZE);
345 long offHeapCacheSize =
346 (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
347 DirectMemoryUtils.getDirectMemorySize());
348 LOG.info("Allocating LruBlockCache with maximum size " +
349 StringUtils.humanReadableInt(cacheSize));
350 if (offHeapCacheSize <= 0) {
351 globalBlockCache = new LruBlockCache(cacheSize,
352 StoreFile.DEFAULT_BLOCKSIZE_SMALL, conf);
353 } else {
354 globalBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize,
355 StoreFile.DEFAULT_BLOCKSIZE_SMALL, blockSize, conf);
356 }
357 return globalBlockCache;
358 }
359 }