1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
22
23 import java.lang.management.ManagementFactory;
24 import java.lang.management.MemoryUsage;
25 import java.util.concurrent.atomic.AtomicLong;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.Chore;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.Server;
34 import org.apache.hadoop.hbase.io.hfile.BlockCache;
35 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
36 import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
37 import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
38 import org.apache.hadoop.hbase.util.Threads;
39 import org.apache.hadoop.util.ReflectionUtils;
40
41 import com.google.common.annotations.VisibleForTesting;
42
43
44
45
46 @InterfaceAudience.Private
47 public class HeapMemoryManager {
48 private static final Log LOG = LogFactory.getLog(HeapMemoryManager.class);
49 private static final int CONVERT_TO_PERCENTAGE = 100;
50 private static final int CLUSTER_MINIMUM_MEMORY_THRESHOLD =
51 (int) (CONVERT_TO_PERCENTAGE * HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD);
52
53 public static final String BLOCK_CACHE_SIZE_MAX_RANGE_KEY = "hfile.block.cache.size.max.range";
54 public static final String BLOCK_CACHE_SIZE_MIN_RANGE_KEY = "hfile.block.cache.size.min.range";
55 public static final String MEMSTORE_SIZE_MAX_RANGE_KEY =
56 "hbase.regionserver.global.memstore.size.max.range";
57 public static final String MEMSTORE_SIZE_MIN_RANGE_KEY =
58 "hbase.regionserver.global.memstore.size.min.range";
59 public static final String HBASE_RS_HEAP_MEMORY_TUNER_PERIOD =
60 "hbase.regionserver.heapmemory.tuner.period";
61 public static final int HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD = 60 * 1000;
62 public static final String HBASE_RS_HEAP_MEMORY_TUNER_CLASS =
63 "hbase.regionserver.heapmemory.tuner.class";
64
65 private float globalMemStorePercent;
66 private float globalMemStorePercentMinRange;
67 private float globalMemStorePercentMaxRange;
68
69 private float blockCachePercent;
70 private float blockCachePercentMinRange;
71 private float blockCachePercentMaxRange;
72 private float l2BlockCachePercent;
73
74 private float heapOccupancyPercent;
75
76 private final ResizableBlockCache blockCache;
77 private final FlushRequester memStoreFlusher;
78 private final Server server;
79
80 private HeapMemoryTunerChore heapMemTunerChore = null;
81 private final boolean tunerOn;
82 private final int defaultChorePeriod;
83 private final float heapOccupancyLowWatermark;
84
85 private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
86
87 public static HeapMemoryManager create(Configuration conf, FlushRequester memStoreFlusher,
88 Server server) {
89 BlockCache blockCache = CacheConfig.instantiateBlockCache(conf);
90 if (blockCache instanceof ResizableBlockCache) {
91 return new HeapMemoryManager((ResizableBlockCache) blockCache, memStoreFlusher, server);
92 }
93 return null;
94 }
95
96 @VisibleForTesting
97 HeapMemoryManager(ResizableBlockCache blockCache, FlushRequester memStoreFlusher,
98 Server server) {
99 Configuration conf = server.getConfiguration();
100 this.blockCache = blockCache;
101 this.memStoreFlusher = memStoreFlusher;
102 this.server = server;
103 this.tunerOn = doInit(conf);
104 this.defaultChorePeriod = conf.getInt(HBASE_RS_HEAP_MEMORY_TUNER_PERIOD,
105 HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD);
106 this.heapOccupancyLowWatermark = conf.getFloat(HConstants.HEAP_OCCUPANCY_LOW_WATERMARK_KEY,
107 HConstants.DEFAULT_HEAP_OCCUPANCY_LOW_WATERMARK);
108 }
109
110 private boolean doInit(Configuration conf) {
111 globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false);
112 blockCachePercent = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY,
113 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
114 HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
115
116 globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
117 globalMemStorePercent);
118 globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
119 globalMemStorePercent);
120 if (globalMemStorePercent < globalMemStorePercentMinRange) {
121 LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " + globalMemStorePercent
122 + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
123 + " because supplied value greater than initial memstore size value.");
124 globalMemStorePercentMinRange = globalMemStorePercent;
125 conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange);
126 }
127 if (globalMemStorePercent > globalMemStorePercentMaxRange) {
128 LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " + globalMemStorePercent
129 + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
130 + " because supplied value less than initial memstore size value.");
131 globalMemStorePercentMaxRange = globalMemStorePercent;
132 conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange);
133 }
134 if (globalMemStorePercent == globalMemStorePercentMinRange
135 && globalMemStorePercent == globalMemStorePercentMaxRange) {
136 return false;
137 }
138
139 blockCachePercentMinRange = conf.getFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercent);
140 blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercent);
141 if (blockCachePercent < blockCachePercentMinRange) {
142 LOG.warn("Setting " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " to " + blockCachePercent
143 + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
144 + " because supplied value greater than initial block cache size.");
145 blockCachePercentMinRange = blockCachePercent;
146 conf.setFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercentMinRange);
147 }
148 if (blockCachePercent > blockCachePercentMaxRange) {
149 LOG.warn("Setting " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " to " + blockCachePercent
150 + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
151 + " because supplied value less than initial block cache size.");
152 blockCachePercentMaxRange = blockCachePercent;
153 conf.setFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercentMaxRange);
154 }
155 if (blockCachePercent == blockCachePercentMinRange
156 && blockCachePercent == blockCachePercentMaxRange) {
157 return false;
158 }
159
160 int gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
161 this.l2BlockCachePercent = HeapMemorySizeUtil.getL2BlockCacheHeapPercent(conf);
162 int bcul = (int) ((blockCachePercentMinRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
163 if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
164 throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
165 + "the threshold required for successful cluster operation. "
166 + "The combined value cannot exceed 0.8. Please check the settings for "
167 + MEMSTORE_SIZE_MAX_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY
168 + " in your configuration. " + MEMSTORE_SIZE_MAX_RANGE_KEY + " is "
169 + globalMemStorePercentMaxRange + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " is "
170 + blockCachePercentMinRange);
171 }
172 gml = (int) (globalMemStorePercentMinRange * CONVERT_TO_PERCENTAGE);
173 bcul = (int) ((blockCachePercentMaxRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
174 if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
175 throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
176 + "the threshold required for successful cluster operation. "
177 + "The combined value cannot exceed 0.8. Please check the settings for "
178 + MEMSTORE_SIZE_MIN_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY
179 + " in your configuration. " + MEMSTORE_SIZE_MIN_RANGE_KEY + " is "
180 + globalMemStorePercentMinRange + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " is "
181 + blockCachePercentMaxRange);
182 }
183 return true;
184 }
185
186 public void start() {
187 LOG.info("Starting HeapMemoryTuner chore.");
188 this.heapMemTunerChore = new HeapMemoryTunerChore();
189 Threads.setDaemonThreadRunning(heapMemTunerChore.getThread());
190 if (tunerOn) {
191
192 memStoreFlusher.registerFlushRequestListener(heapMemTunerChore);
193 }
194 }
195
196 public void stop() {
197
198 LOG.info("Stoping HeapMemoryTuner chore.");
199 this.heapMemTunerChore.interrupt();
200 }
201
202
203 boolean isTunerOn() {
204 return this.tunerOn;
205 }
206
207
208
209
210 public float getHeapOccupancyPercent() {
211 return this.heapOccupancyPercent;
212 }
213
214 private class HeapMemoryTunerChore extends Chore implements FlushRequestListener {
215 private HeapMemoryTuner heapMemTuner;
216 private AtomicLong blockedFlushCount = new AtomicLong();
217 private AtomicLong unblockedFlushCount = new AtomicLong();
218 private long evictCount = 0L;
219 private TunerContext tunerContext = new TunerContext();
220 private boolean alarming = false;
221
222 public HeapMemoryTunerChore() {
223 super(server.getServerName() + "-HeapMemoryTunerChore", defaultChorePeriod, server);
224 Class<? extends HeapMemoryTuner> tunerKlass = server.getConfiguration().getClass(
225 HBASE_RS_HEAP_MEMORY_TUNER_CLASS, DefaultHeapMemoryTuner.class, HeapMemoryTuner.class);
226 heapMemTuner = ReflectionUtils.newInstance(tunerKlass, server.getConfiguration());
227 }
228
229 @Override
230 protected void sleep() {
231 if (!alarming) {
232 super.sleep();
233 } else {
234
235 try {
236 Thread.sleep(1000);
237 } catch (InterruptedException e) {
238
239 Thread.currentThread().interrupt();
240 }
241 }
242 }
243
244 @Override
245 protected void chore() {
246
247 MemoryUsage memUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
248 heapOccupancyPercent = (float)memUsage.getUsed() / (float)memUsage.getCommitted();
249
250
251 if (heapOccupancyPercent >= heapOccupancyLowWatermark) {
252 if (!alarming) {
253 LOG.warn("heapOccupancyPercent " + heapOccupancyPercent +
254 " is above heap occupancy alarm watermark (" + heapOccupancyLowWatermark + ")");
255 alarming = true;
256 }
257 } else {
258 if (alarming) {
259 LOG.info("heapOccupancyPercent " + heapOccupancyPercent +
260 " is now below the heap occupancy alarm watermark (" +
261 heapOccupancyLowWatermark + ")");
262 alarming = false;
263 }
264 }
265
266 if (tunerOn && !alarming) {
267 tune();
268 }
269 }
270
271 private void tune() {
272 evictCount = blockCache.getStats().getEvictedCount() - evictCount;
273 tunerContext.setBlockedFlushCount(blockedFlushCount.getAndSet(0));
274 tunerContext.setUnblockedFlushCount(unblockedFlushCount.getAndSet(0));
275 tunerContext.setEvictCount(evictCount);
276 tunerContext.setCurBlockCacheSize(blockCachePercent);
277 tunerContext.setCurMemStoreSize(globalMemStorePercent);
278 TunerResult result = null;
279 try {
280 result = this.heapMemTuner.tune(tunerContext);
281 } catch (Throwable t) {
282 LOG.error("Exception thrown from the HeapMemoryTuner implementation", t);
283 }
284 if (result != null && result.needsTuning()) {
285 float memstoreSize = result.getMemstoreSize();
286 float blockCacheSize = result.getBlockCacheSize();
287 LOG.debug("From HeapMemoryTuner new memstoreSize: " + memstoreSize
288 + ". new blockCacheSize: " + blockCacheSize);
289 if (memstoreSize < globalMemStorePercentMinRange) {
290 LOG.info("New memstoreSize from HeapMemoryTuner " + memstoreSize + " is below min level "
291 + globalMemStorePercentMinRange + ". Resetting memstoreSize to min size");
292 memstoreSize = globalMemStorePercentMinRange;
293 } else if (memstoreSize > globalMemStorePercentMaxRange) {
294 LOG.info("New memstoreSize from HeapMemoryTuner " + memstoreSize + " is above max level "
295 + globalMemStorePercentMaxRange + ". Resetting memstoreSize to max size");
296 memstoreSize = globalMemStorePercentMaxRange;
297 }
298 if (blockCacheSize < blockCachePercentMinRange) {
299 LOG.info("New blockCacheSize from HeapMemoryTuner " + blockCacheSize
300 + " is below min level " + blockCachePercentMinRange
301 + ". Resetting blockCacheSize to min size");
302 blockCacheSize = blockCachePercentMinRange;
303 } else if (blockCacheSize > blockCachePercentMaxRange) {
304 LOG.info("New blockCacheSize from HeapMemoryTuner " + blockCacheSize
305 + " is above max level " + blockCachePercentMaxRange
306 + ". Resetting blockCacheSize to min size");
307 blockCacheSize = blockCachePercentMaxRange;
308 }
309 int gml = (int) (memstoreSize * CONVERT_TO_PERCENTAGE);
310 int bcul = (int) ((blockCacheSize + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
311 if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
312 LOG.info("Current heap configuration from HeapMemoryTuner exceeds "
313 + "the threshold required for successful cluster operation. "
314 + "The combined value cannot exceed 0.8. " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
315 + " is " + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
316 + blockCacheSize);
317
318 } else {
319 long newBlockCacheSize = (long) (maxHeapSize * blockCacheSize);
320 long newMemstoreSize = (long) (maxHeapSize * memstoreSize);
321 LOG.info("Setting block cache heap size to " + newBlockCacheSize
322 + " and memstore heap size to " + newMemstoreSize);
323 blockCachePercent = blockCacheSize;
324 blockCache.setMaxSize(newBlockCacheSize);
325 globalMemStorePercent = memstoreSize;
326 memStoreFlusher.setGlobalMemstoreLimit(newMemstoreSize);
327 }
328 }
329 }
330
331 @Override
332 public void flushRequested(FlushType type, HRegion region) {
333 switch (type) {
334 case ABOVE_HIGHER_MARK:
335 blockedFlushCount.incrementAndGet();
336 break;
337 case ABOVE_LOWER_MARK:
338 unblockedFlushCount.incrementAndGet();
339 break;
340 default:
341
342 break;
343 }
344 }
345 }
346
347
348
349
350
351
352 public static final class TunerContext {
353 private long blockedFlushCount;
354 private long unblockedFlushCount;
355 private long evictCount;
356 private float curMemStoreSize;
357 private float curBlockCacheSize;
358
359 public long getBlockedFlushCount() {
360 return blockedFlushCount;
361 }
362
363 public void setBlockedFlushCount(long blockedFlushCount) {
364 this.blockedFlushCount = blockedFlushCount;
365 }
366
367 public long getUnblockedFlushCount() {
368 return unblockedFlushCount;
369 }
370
371 public void setUnblockedFlushCount(long unblockedFlushCount) {
372 this.unblockedFlushCount = unblockedFlushCount;
373 }
374
375 public long getEvictCount() {
376 return evictCount;
377 }
378
379 public void setEvictCount(long evictCount) {
380 this.evictCount = evictCount;
381 }
382
383 public float getCurMemStoreSize() {
384 return curMemStoreSize;
385 }
386
387 public void setCurMemStoreSize(float curMemStoreSize) {
388 this.curMemStoreSize = curMemStoreSize;
389 }
390
391 public float getCurBlockCacheSize() {
392 return curBlockCacheSize;
393 }
394
395 public void setCurBlockCacheSize(float curBlockCacheSize) {
396 this.curBlockCacheSize = curBlockCacheSize;
397 }
398 }
399
400
401
402
403
404 public static final class TunerResult {
405 private float memstoreSize;
406 private float blockCacheSize;
407 private final boolean needsTuning;
408
409 public TunerResult(boolean needsTuning) {
410 this.needsTuning = needsTuning;
411 }
412
413 public float getMemstoreSize() {
414 return memstoreSize;
415 }
416
417 public void setMemstoreSize(float memstoreSize) {
418 this.memstoreSize = memstoreSize;
419 }
420
421 public float getBlockCacheSize() {
422 return blockCacheSize;
423 }
424
425 public void setBlockCacheSize(float blockCacheSize) {
426 this.blockCacheSize = blockCacheSize;
427 }
428
429 public boolean needsTuning() {
430 return needsTuning;
431 }
432 }
433 }