View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
22  
23  import java.lang.management.ManagementFactory;
24  import java.lang.management.MemoryUsage;
25  import java.util.concurrent.atomic.AtomicLong;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.classification.InterfaceAudience;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.Chore;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.Server;
34  import org.apache.hadoop.hbase.io.hfile.BlockCache;
35  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
36  import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
37  import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
38  import org.apache.hadoop.hbase.util.Threads;
39  import org.apache.hadoop.util.ReflectionUtils;
40  
41  import com.google.common.annotations.VisibleForTesting;
42  
43  /**
44   * Manages tuning of Heap memory using <code>HeapMemoryTuner</code>.
45   */
46  @InterfaceAudience.Private
47  public class HeapMemoryManager {
48    private static final Log LOG = LogFactory.getLog(HeapMemoryManager.class);
49    private static final int CONVERT_TO_PERCENTAGE = 100;
50    private static final int CLUSTER_MINIMUM_MEMORY_THRESHOLD = 
51      (int) (CONVERT_TO_PERCENTAGE * HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD);
52  
53    public static final String BLOCK_CACHE_SIZE_MAX_RANGE_KEY = "hfile.block.cache.size.max.range";
54    public static final String BLOCK_CACHE_SIZE_MIN_RANGE_KEY = "hfile.block.cache.size.min.range";
55    public static final String MEMSTORE_SIZE_MAX_RANGE_KEY = 
56        "hbase.regionserver.global.memstore.size.max.range";
57    public static final String MEMSTORE_SIZE_MIN_RANGE_KEY = 
58        "hbase.regionserver.global.memstore.size.min.range";
59    public static final String HBASE_RS_HEAP_MEMORY_TUNER_PERIOD = 
60        "hbase.regionserver.heapmemory.tuner.period";
61    public static final int HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD = 60 * 1000;
62    public static final String HBASE_RS_HEAP_MEMORY_TUNER_CLASS = 
63        "hbase.regionserver.heapmemory.tuner.class";
64  
65    private float globalMemStorePercent;
66    private float globalMemStorePercentMinRange;
67    private float globalMemStorePercentMaxRange;
68  
69    private float blockCachePercent;
70    private float blockCachePercentMinRange;
71    private float blockCachePercentMaxRange;
72    private float l2BlockCachePercent;
73  
74    private float heapOccupancyPercent;
75  
76    private final ResizableBlockCache blockCache;
77    private final FlushRequester memStoreFlusher;
78    private final Server server;
79  
80    private HeapMemoryTunerChore heapMemTunerChore = null;
81    private final boolean tunerOn;
82    private final int defaultChorePeriod;
83    private final float heapOccupancyLowWatermark;
84  
85    private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
86  
87    public static HeapMemoryManager create(Configuration conf, FlushRequester memStoreFlusher,
88        Server server) {
89      BlockCache blockCache = CacheConfig.instantiateBlockCache(conf);
90      if (blockCache instanceof ResizableBlockCache) {
91        return new HeapMemoryManager((ResizableBlockCache) blockCache, memStoreFlusher, server);
92      }
93      return null;
94    }
95  
96    @VisibleForTesting
97    HeapMemoryManager(ResizableBlockCache blockCache, FlushRequester memStoreFlusher,
98        Server server) {
99      Configuration conf = server.getConfiguration();
100     this.blockCache = blockCache;
101     this.memStoreFlusher = memStoreFlusher;
102     this.server = server;
103     this.tunerOn = doInit(conf);
104     this.defaultChorePeriod = conf.getInt(HBASE_RS_HEAP_MEMORY_TUNER_PERIOD,
105       HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD);
106     this.heapOccupancyLowWatermark = conf.getFloat(HConstants.HEAP_OCCUPANCY_LOW_WATERMARK_KEY,
107       HConstants.DEFAULT_HEAP_OCCUPANCY_LOW_WATERMARK);
108   }
109 
110   private boolean doInit(Configuration conf) {
111     globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false);
112     blockCachePercent = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY,
113         HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
114     HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
115     // Initialize max and min range for memstore heap space
116     globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
117         globalMemStorePercent);
118     globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
119         globalMemStorePercent);
120     if (globalMemStorePercent < globalMemStorePercentMinRange) {
121       LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " + globalMemStorePercent
122           + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
123           + " because supplied value greater than initial memstore size value.");
124       globalMemStorePercentMinRange = globalMemStorePercent;
125       conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange);
126     }
127     if (globalMemStorePercent > globalMemStorePercentMaxRange) {
128       LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " + globalMemStorePercent
129           + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
130           + " because supplied value less than initial memstore size value.");
131       globalMemStorePercentMaxRange = globalMemStorePercent;
132       conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange);
133     }
134     if (globalMemStorePercent == globalMemStorePercentMinRange
135         && globalMemStorePercent == globalMemStorePercentMaxRange) {
136       return false;
137     }
138     // Initialize max and min range for block cache
139     blockCachePercentMinRange = conf.getFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercent);
140     blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercent);
141     if (blockCachePercent < blockCachePercentMinRange) {
142       LOG.warn("Setting " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " to " + blockCachePercent
143           + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
144           + " because supplied value greater than initial block cache size.");
145       blockCachePercentMinRange = blockCachePercent;
146       conf.setFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercentMinRange);
147     }
148     if (blockCachePercent > blockCachePercentMaxRange) {
149       LOG.warn("Setting " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " to " + blockCachePercent
150           + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
151           + " because supplied value less than initial block cache size.");
152       blockCachePercentMaxRange = blockCachePercent;
153       conf.setFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercentMaxRange);
154     }
155     if (blockCachePercent == blockCachePercentMinRange
156         && blockCachePercent == blockCachePercentMaxRange) {
157       return false;
158     }
159 
160     int gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
161     this.l2BlockCachePercent = HeapMemorySizeUtil.getL2BlockCacheHeapPercent(conf);
162     int bcul = (int) ((blockCachePercentMinRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
163     if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
164       throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
165           + "the threshold required for successful cluster operation. "
166           + "The combined value cannot exceed 0.8. Please check the settings for "
167           + MEMSTORE_SIZE_MAX_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY
168           + " in your configuration. " + MEMSTORE_SIZE_MAX_RANGE_KEY + " is "
169           + globalMemStorePercentMaxRange + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " is "
170           + blockCachePercentMinRange);
171     }
172     gml = (int) (globalMemStorePercentMinRange * CONVERT_TO_PERCENTAGE);
173     bcul = (int) ((blockCachePercentMaxRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
174     if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
175       throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
176           + "the threshold required for successful cluster operation. "
177           + "The combined value cannot exceed 0.8. Please check the settings for "
178           + MEMSTORE_SIZE_MIN_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY
179           + " in your configuration. " + MEMSTORE_SIZE_MIN_RANGE_KEY + " is "
180           + globalMemStorePercentMinRange + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " is "
181           + blockCachePercentMaxRange);
182     }
183     return true;
184   }
185 
186   public void start() {
187     LOG.info("Starting HeapMemoryTuner chore.");
188     this.heapMemTunerChore = new HeapMemoryTunerChore();
189     Threads.setDaemonThreadRunning(heapMemTunerChore.getThread());
190     if (tunerOn) {
191       // Register HeapMemoryTuner as a memstore flush listener
192       memStoreFlusher.registerFlushRequestListener(heapMemTunerChore);
193     }
194   }
195 
196   public void stop() {
197     // The thread is Daemon. Just interrupting the ongoing process.
198     LOG.info("Stoping HeapMemoryTuner chore.");
199     this.heapMemTunerChore.interrupt();
200   }
201 
202   // Used by the test cases.
203   boolean isTunerOn() {
204     return this.tunerOn;
205   }
206 
207   /**
208    * @return heap occupancy percentage, 0 <= n <= 1
209    */
210   public float getHeapOccupancyPercent() {
211     return this.heapOccupancyPercent;
212   }
213 
214   private class HeapMemoryTunerChore extends Chore implements FlushRequestListener {
215     private HeapMemoryTuner heapMemTuner;
216     private AtomicLong blockedFlushCount = new AtomicLong();
217     private AtomicLong unblockedFlushCount = new AtomicLong();
218     private long evictCount = 0L;
219     private TunerContext tunerContext = new TunerContext();
220     private boolean alarming = false;
221 
222     public HeapMemoryTunerChore() {
223       super(server.getServerName() + "-HeapMemoryTunerChore", defaultChorePeriod, server);
224       Class<? extends HeapMemoryTuner> tunerKlass = server.getConfiguration().getClass(
225           HBASE_RS_HEAP_MEMORY_TUNER_CLASS, DefaultHeapMemoryTuner.class, HeapMemoryTuner.class);
226       heapMemTuner = ReflectionUtils.newInstance(tunerKlass, server.getConfiguration());
227     }
228 
229     @Override
230     protected void sleep() {
231       if (!alarming) {
232         super.sleep();
233       } else {
234         // we are in the alarm state, so sleep only for a short fixed period
235         try {
236           Thread.sleep(1000);
237         } catch (InterruptedException e) {
238           // Interrupted, propagate
239           Thread.currentThread().interrupt();
240         }
241       }
242     }
243 
244     @Override
245     protected void chore() {
246       // Sample heap occupancy
247       MemoryUsage memUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
248       heapOccupancyPercent = (float)memUsage.getUsed() / (float)memUsage.getCommitted();
249       // If we are above the heap occupancy alarm low watermark, switch to short
250       // sleeps for close monitoring. Stop autotuning, we are in a danger zone.
251       if (heapOccupancyPercent >= heapOccupancyLowWatermark) {
252         if (!alarming) {
253           LOG.warn("heapOccupancyPercent " + heapOccupancyPercent +
254             " is above heap occupancy alarm watermark (" + heapOccupancyLowWatermark + ")");
255           alarming = true;
256         }
257       } else {
258         if (alarming) {
259           LOG.info("heapOccupancyPercent " + heapOccupancyPercent +
260             " is now below the heap occupancy alarm watermark (" +
261             heapOccupancyLowWatermark + ")");
262           alarming = false;
263         }
264       }
265       // Autotune if tuning is enabled and allowed
266       if (tunerOn && !alarming) {
267         tune();
268       }
269     }
270 
271     private void tune() {
272       evictCount = blockCache.getStats().getEvictedCount() - evictCount;
273       tunerContext.setBlockedFlushCount(blockedFlushCount.getAndSet(0));
274       tunerContext.setUnblockedFlushCount(unblockedFlushCount.getAndSet(0));
275       tunerContext.setEvictCount(evictCount);
276       tunerContext.setCurBlockCacheSize(blockCachePercent);
277       tunerContext.setCurMemStoreSize(globalMemStorePercent);
278       TunerResult result = null;
279       try {
280         result = this.heapMemTuner.tune(tunerContext);
281       } catch (Throwable t) {
282         LOG.error("Exception thrown from the HeapMemoryTuner implementation", t);
283       }
284       if (result != null && result.needsTuning()) {
285         float memstoreSize = result.getMemstoreSize();
286         float blockCacheSize = result.getBlockCacheSize();
287         LOG.debug("From HeapMemoryTuner new memstoreSize: " + memstoreSize
288             + ". new blockCacheSize: " + blockCacheSize);
289         if (memstoreSize < globalMemStorePercentMinRange) {
290           LOG.info("New memstoreSize from HeapMemoryTuner " + memstoreSize + " is below min level "
291               + globalMemStorePercentMinRange + ". Resetting memstoreSize to min size");
292           memstoreSize = globalMemStorePercentMinRange;
293         } else if (memstoreSize > globalMemStorePercentMaxRange) {
294           LOG.info("New memstoreSize from HeapMemoryTuner " + memstoreSize + " is above max level "
295               + globalMemStorePercentMaxRange + ". Resetting memstoreSize to max size");
296           memstoreSize = globalMemStorePercentMaxRange;
297         }
298         if (blockCacheSize < blockCachePercentMinRange) {
299           LOG.info("New blockCacheSize from HeapMemoryTuner " + blockCacheSize
300               + " is below min level " + blockCachePercentMinRange
301               + ". Resetting blockCacheSize to min size");
302           blockCacheSize = blockCachePercentMinRange;
303         } else if (blockCacheSize > blockCachePercentMaxRange) {
304           LOG.info("New blockCacheSize from HeapMemoryTuner " + blockCacheSize
305               + " is above max level " + blockCachePercentMaxRange
306               + ". Resetting blockCacheSize to min size");
307           blockCacheSize = blockCachePercentMaxRange;
308         }
309         int gml = (int) (memstoreSize * CONVERT_TO_PERCENTAGE);
310         int bcul = (int) ((blockCacheSize + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
311         if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
312           LOG.info("Current heap configuration from HeapMemoryTuner exceeds "
313               + "the threshold required for successful cluster operation. "
314               + "The combined value cannot exceed 0.8. " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
315               + " is " + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
316               + blockCacheSize);
317           // TODO can adjust the value so as not exceed 80%. Is that correct? may be.
318         } else {
319           long newBlockCacheSize = (long) (maxHeapSize * blockCacheSize);
320           long newMemstoreSize = (long) (maxHeapSize * memstoreSize);
321           LOG.info("Setting block cache heap size to " + newBlockCacheSize
322               + " and memstore heap size to " + newMemstoreSize);
323           blockCachePercent = blockCacheSize;
324           blockCache.setMaxSize(newBlockCacheSize);
325           globalMemStorePercent = memstoreSize;
326           memStoreFlusher.setGlobalMemstoreLimit(newMemstoreSize);
327         }
328       }
329     }
330 
331     @Override
332     public void flushRequested(FlushType type, HRegion region) {
333       switch (type) {
334       case ABOVE_HIGHER_MARK:
335         blockedFlushCount.incrementAndGet();
336         break;
337       case ABOVE_LOWER_MARK:
338         unblockedFlushCount.incrementAndGet();
339         break;
340       default:
341         // In case of normal flush don't do any action.
342         break;
343       }
344     }
345   }
346 
347   /**
348    * POJO to pass all the relevant information required to do the heap memory tuning. It holds the
349    * flush counts and block cache evictions happened within the interval. Also holds the current
350    * heap percentage allocated for memstore and block cache.
351    */
352   public static final class TunerContext {
353     private long blockedFlushCount;
354     private long unblockedFlushCount;
355     private long evictCount;
356     private float curMemStoreSize;
357     private float curBlockCacheSize;
358 
359     public long getBlockedFlushCount() {
360       return blockedFlushCount;
361     }
362 
363     public void setBlockedFlushCount(long blockedFlushCount) {
364       this.blockedFlushCount = blockedFlushCount;
365     }
366 
367     public long getUnblockedFlushCount() {
368       return unblockedFlushCount;
369     }
370 
371     public void setUnblockedFlushCount(long unblockedFlushCount) {
372       this.unblockedFlushCount = unblockedFlushCount;
373     }
374 
375     public long getEvictCount() {
376       return evictCount;
377     }
378 
379     public void setEvictCount(long evictCount) {
380       this.evictCount = evictCount;
381     }
382 
383     public float getCurMemStoreSize() {
384       return curMemStoreSize;
385     }
386 
387     public void setCurMemStoreSize(float curMemStoreSize) {
388       this.curMemStoreSize = curMemStoreSize;
389     }
390 
391     public float getCurBlockCacheSize() {
392       return curBlockCacheSize;
393     }
394 
395     public void setCurBlockCacheSize(float curBlockCacheSize) {
396       this.curBlockCacheSize = curBlockCacheSize;
397     }
398   }
399 
400   /**
401    * POJO which holds the result of memory tuning done by HeapMemoryTuner implementation.
402    * It includes the new heap percentage for memstore and block cache.
403    */
404   public static final class TunerResult {
405     private float memstoreSize;
406     private float blockCacheSize;
407     private final boolean needsTuning;
408 
409     public TunerResult(boolean needsTuning) {
410       this.needsTuning = needsTuning;
411     }
412 
413     public float getMemstoreSize() {
414       return memstoreSize;
415     }
416 
417     public void setMemstoreSize(float memstoreSize) {
418       this.memstoreSize = memstoreSize;
419     }
420 
421     public float getBlockCacheSize() {
422       return blockCacheSize;
423     }
424 
425     public void setBlockCacheSize(float blockCacheSize) {
426       this.blockCacheSize = blockCacheSize;
427     }
428 
429     public boolean needsTuning() {
430       return needsTuning;
431     }
432   }
433 }